HMVMXR0.cpp@ 70161

Last change on this file since 70161 was 69856, checked in by vboxsync, 7 years ago
VMM/HMVMXR0: Fix assertion on longjmp callback code path while saving the guest state.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 597.7 KB

Line
1	/* $Id: HMVMXR0.cpp 69856 2017-11-28 10:00:14Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2017 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#include <iprt/x86.h>
24	#include <iprt/asm-amd64-x86.h>
25	#include <iprt/thread.h>
26
27	#include <VBox/vmm/pdmapi.h>
28	#include <VBox/vmm/dbgf.h>
29	#include <VBox/vmm/iem.h>
30	#include <VBox/vmm/iom.h>
31	#include <VBox/vmm/selm.h>
32	#include <VBox/vmm/tm.h>
33	#include <VBox/vmm/gim.h>
34	#include <VBox/vmm/apic.h>
35	#ifdef VBOX_WITH_REM
36	# include <VBox/vmm/rem.h>
37	#endif
38	#include "HMInternal.h"
39	#include <VBox/vmm/vm.h>
40	#include "HMVMXR0.h"
41	#include "dtrace/VBoxVMM.h"
42
43	#define HMVMX_USE_IEM_EVENT_REFLECTION
44	#ifdef DEBUG_ramshankar
45	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
47	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
48	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
49	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
50	# define HMVMX_ALWAYS_TRAP_PF
51	# define HMVMX_ALWAYS_SWAP_FPU_STATE
52	# define HMVMX_ALWAYS_FLUSH_TLB
53	# define HMVMX_ALWAYS_SWAP_EFER
54	#endif
55
56
57	/*********************************************************************************************************************************
58	* Defined Constants And Macros *
59	*********************************************************************************************************************************/
60	/** Use the function table. */
61	#define HMVMX_USE_FUNCTION_TABLE
62
63	/** Determine which tagged-TLB flush handler to use. */
64	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
66	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
67	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
68
69	/** @name Updated-guest-state flags.
70	* @{ */
71	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
72	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
73	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
74	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
75	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
76	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
77	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
78	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
79	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
80	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
81	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
82	#define HMVMX_UPDATED_GUEST_DR7 RT_BIT(11)
83	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
84	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
85	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
86	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
87	#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
88	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
89	#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
90	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
91	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
92	\| HMVMX_UPDATED_GUEST_RSP \
93	\| HMVMX_UPDATED_GUEST_RFLAGS \
94	\| HMVMX_UPDATED_GUEST_CR0 \
95	\| HMVMX_UPDATED_GUEST_CR3 \
96	\| HMVMX_UPDATED_GUEST_CR4 \
97	\| HMVMX_UPDATED_GUEST_GDTR \
98	\| HMVMX_UPDATED_GUEST_IDTR \
99	\| HMVMX_UPDATED_GUEST_LDTR \
100	\| HMVMX_UPDATED_GUEST_TR \
101	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
102	\| HMVMX_UPDATED_GUEST_DR7 \
103	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
104	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
105	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
106	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
107	\| HMVMX_UPDATED_GUEST_LAZY_MSRS \
108	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
109	\| HMVMX_UPDATED_GUEST_INTR_STATE \
110	\| HMVMX_UPDATED_GUEST_APIC_STATE)
111	/** @} */
112
113	/** @name
114	* Flags to skip redundant reads of some common VMCS fields that are not part of
115	* the guest-CPU state but are in the transient structure.
116	*/
117	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
118	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
119	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
120	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
124	/** @} */
125
126	/** @name
127	* States of the VMCS.
128	*
129	* This does not reflect all possible VMCS states but currently only those
130	* needed for maintaining the VMCS consistently even when thread-context hooks
131	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
132	*/
133	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
134	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
135	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
136	/** @} */
137
138	/**
139	* Exception bitmap mask for real-mode guests (real-on-v86).
140	*
141	* We need to intercept all exceptions manually except:
142	* - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
143	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
144	* due to bugs in Intel CPUs.
145	* - \#PF need not be intercepted even in real-mode if we have Nested Paging
146	* support.
147	*/
148	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
149	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
150	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
151	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
152	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
153	/* RT_BIT(X86_XCPT_MF) always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
154	\| RT_BIT(X86_XCPT_XF))
155
156	/**
157	* Exception bitmap mask for all contributory exceptions.
158	*
159	* Page fault is deliberately excluded here as it's conditional as to whether
160	* it's contributory or benign. Page faults are handled separately.
161	*/
162	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
163	\| RT_BIT(X86_XCPT_DE))
164
165	/** Maximum VM-instruction error number. */
166	#define HMVMX_INSTR_ERROR_MAX 28
167
168	/** Profiling macro. */
169	#ifdef HM_PROFILE_EXIT_DISPATCH
170	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
171	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
172	#else
173	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
174	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
175	#endif
176
177	/** Assert that preemption is disabled or covered by thread-context hooks. */
178	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
179	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
180
181	/** Assert that we haven't migrated CPUs when thread-context hooks are not
182	* used. */
183	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
184	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
185	("Illegal migration! Entered on CPU %u Current %u\n", \
186	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
187
188	/** Helper macro for VM-exit handlers called unexpectedly. */
189	#define HMVMX_RETURN_UNEXPECTED_EXIT() \
190	do { \
191	pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
192	return VERR_VMX_UNEXPECTED_EXIT; \
193	} while (0)
194
195
196	/*********************************************************************************************************************************
197	* Structures and Typedefs *
198	*********************************************************************************************************************************/
199	/**
200	* VMX transient state.
201	*
202	* A state structure for holding miscellaneous information across
203	* VMX non-root operation and restored after the transition.
204	*/
205	typedef struct VMXTRANSIENT
206	{
207	/** The host's rflags/eflags. */
208	RTCCUINTREG fEFlags;
209	#if HC_ARCH_BITS == 32
210	uint32_t u32Alignment0;
211	#endif
212	/** The guest's TPR value used for TPR shadowing. */
213	uint8_t u8GuestTpr;
214	/** Alignment. */
215	uint8_t abAlignment0[7];
216
217	/** The basic VM-exit reason. */
218	uint16_t uExitReason;
219	/** Alignment. */
220	uint16_t u16Alignment0;
221	/** The VM-exit interruption error code. */
222	uint32_t uExitIntErrorCode;
223	/** The VM-exit exit code qualification. */
224	uint64_t uExitQualification;
225
226	/** The VM-exit interruption-information field. */
227	uint32_t uExitIntInfo;
228	/** The VM-exit instruction-length field. */
229	uint32_t cbInstr;
230	/** The VM-exit instruction-information field. */
231	union
232	{
233	/** Plain unsigned int representation. */
234	uint32_t u;
235	/** INS and OUTS information. */
236	struct
237	{
238	uint32_t u7Reserved0 : 7;
239	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
240	uint32_t u3AddrSize : 3;
241	uint32_t u5Reserved1 : 5;
242	/** The segment register (X86_SREG_XXX). */
243	uint32_t iSegReg : 3;
244	uint32_t uReserved2 : 14;
245	} StrIo;
246	} ExitInstrInfo;
247	/** Whether the VM-entry failed or not. */
248	bool fVMEntryFailed;
249	/** Alignment. */
250	uint8_t abAlignment1[3];
251
252	/** The VM-entry interruption-information field. */
253	uint32_t uEntryIntInfo;
254	/** The VM-entry exception error code field. */
255	uint32_t uEntryXcptErrorCode;
256	/** The VM-entry instruction length field. */
257	uint32_t cbEntryInstr;
258
259	/** IDT-vectoring information field. */
260	uint32_t uIdtVectoringInfo;
261	/** IDT-vectoring error code. */
262	uint32_t uIdtVectoringErrorCode;
263
264	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
265	uint32_t fVmcsFieldsRead;
266
267	/** Whether the guest FPU was active at the time of VM-exit. */
268	bool fWasGuestFPUStateActive;
269	/** Whether the guest debug state was active at the time of VM-exit. */
270	bool fWasGuestDebugStateActive;
271	/** Whether the hyper debug state was active at the time of VM-exit. */
272	bool fWasHyperDebugStateActive;
273	/** Whether TSC-offsetting should be setup before VM-entry. */
274	bool fUpdateTscOffsettingAndPreemptTimer;
275	/** Whether the VM-exit was caused by a page-fault during delivery of a
276	* contributory exception or a page-fault. */
277	bool fVectoringDoublePF;
278	/** Whether the VM-exit was caused by a page-fault during delivery of an
279	* external interrupt or NMI. */
280	bool fVectoringPF;
281	} VMXTRANSIENT;
282	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
283	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
284	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
285	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
286	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
287	/** Pointer to VMX transient state. */
288	typedef VMXTRANSIENT *PVMXTRANSIENT;
289
290
291	/**
292	* MSR-bitmap read permissions.
293	*/
294	typedef enum VMXMSREXITREAD
295	{
296	/** Reading this MSR causes a VM-exit. */
297	VMXMSREXIT_INTERCEPT_READ = 0xb,
298	/** Reading this MSR does not cause a VM-exit. */
299	VMXMSREXIT_PASSTHRU_READ
300	} VMXMSREXITREAD;
301	/** Pointer to MSR-bitmap read permissions. */
302	typedef VMXMSREXITREAD* PVMXMSREXITREAD;
303
304	/**
305	* MSR-bitmap write permissions.
306	*/
307	typedef enum VMXMSREXITWRITE
308	{
309	/** Writing to this MSR causes a VM-exit. */
310	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
311	/** Writing to this MSR does not cause a VM-exit. */
312	VMXMSREXIT_PASSTHRU_WRITE
313	} VMXMSREXITWRITE;
314	/** Pointer to MSR-bitmap write permissions. */
315	typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
316
317
318	/**
319	* VMX VM-exit handler.
320	*
321	* @returns Strict VBox status code (i.e. informational status codes too).
322	* @param pVCpu The cross context virtual CPU structure.
323	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
324	* out-of-sync. Make sure to update the required
325	* fields before using them.
326	* @param pVmxTransient Pointer to the VMX-transient structure.
327	*/
328	#ifndef HMVMX_USE_FUNCTION_TABLE
329	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
330	#else
331	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
332	/** Pointer to VM-exit handler. */
333	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
334	#endif
335
336	/**
337	* VMX VM-exit handler, non-strict status code.
338	*
339	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
340	*
341	* @returns VBox status code, no informational status code returned.
342	* @param pVCpu The cross context virtual CPU structure.
343	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
344	* out-of-sync. Make sure to update the required
345	* fields before using them.
346	* @param pVmxTransient Pointer to the VMX-transient structure.
347	*
348	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
349	* use of that status code will be replaced with VINF_EM_SOMETHING
350	* later when switching over to IEM.
351	*/
352	#ifndef HMVMX_USE_FUNCTION_TABLE
353	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
354	#else
355	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
356	#endif
357
358
359	/*********************************************************************************************************************************
360	* Internal Functions *
361	*********************************************************************************************************************************/
362	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
363	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
364	static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
365	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
366	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
367	bool fStepping, uint32_t *puIntState);
368	#if HC_ARCH_BITS == 32
369	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
370	#endif
371	#ifndef HMVMX_USE_FUNCTION_TABLE
372	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
373	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
374	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
375	#else
376	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
377	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
378	#endif
379
380
381	/** @name VM-exit handlers.
382	* @{
383	*/
384	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
385	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
386	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
387	static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
388	static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
389	static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
390	static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
391	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
392	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
393	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
394	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
395	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
396	static FNVMXEXITHANDLER hmR0VmxExitHlt;
397	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
398	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
399	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
400	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
401	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
402	static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
403	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
404	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
405	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
406	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
407	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
408	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
409	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
410	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
411	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
412	static FNVMXEXITHANDLER hmR0VmxExitMwait;
413	static FNVMXEXITHANDLER hmR0VmxExitMtf;
414	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
415	static FNVMXEXITHANDLER hmR0VmxExitPause;
416	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
417	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
418	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
419	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
420	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
421	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
422	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
423	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
424	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
425	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
426	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
427	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
428	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
429	/** @} */
430
431	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
432	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
433	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
434	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
435	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
436	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
437	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
438	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
439	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
440
441
442	/*********************************************************************************************************************************
443	* Global Variables *
444	*********************************************************************************************************************************/
445	#ifdef HMVMX_USE_FUNCTION_TABLE
446
447	/**
448	* VMX_EXIT dispatch table.
449	*/
450	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
451	{
452	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
453	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
454	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
455	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
456	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
457	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
458	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
459	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
460	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
461	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
462	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
463	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
464	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
465	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
466	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
467	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
468	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
469	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
470	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
471	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
472	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
473	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
474	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
475	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
476	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
477	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
478	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
479	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
480	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
481	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
482	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
483	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
484	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
485	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
486	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
487	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
488	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
489	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
490	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
491	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
492	/* 40 UNDEFINED */ hmR0VmxExitPause,
493	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
494	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
495	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
496	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
497	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
498	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
499	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
500	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
501	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
502	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
503	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
504	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
505	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
506	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
507	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
508	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
509	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
510	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
511	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
512	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
513	/* 61 VMX_EXIT_RDSEED / hmR0VmxExitErrUndefined, / only spurious exits, so undefined */
514	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
515	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
516	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
517	};
518	#endif /* HMVMX_USE_FUNCTION_TABLE */
519
520	#ifdef VBOX_STRICT
521	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
522	{
523	/* 0 */ "(Not Used)",
524	/* 1 */ "VMCALL executed in VMX root operation.",
525	/* 2 */ "VMCLEAR with invalid physical address.",
526	/* 3 */ "VMCLEAR with VMXON pointer.",
527	/* 4 */ "VMLAUNCH with non-clear VMCS.",
528	/* 5 */ "VMRESUME with non-launched VMCS.",
529	/* 6 */ "VMRESUME after VMXOFF",
530	/* 7 */ "VM-entry with invalid control fields.",
531	/* 8 */ "VM-entry with invalid host state fields.",
532	/* 9 */ "VMPTRLD with invalid physical address.",
533	/* 10 */ "VMPTRLD with VMXON pointer.",
534	/* 11 */ "VMPTRLD with incorrect revision identifier.",
535	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
536	/* 13 */ "VMWRITE to read-only VMCS component.",
537	/* 14 */ "(Not Used)",
538	/* 15 */ "VMXON executed in VMX root operation.",
539	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
540	/* 17 */ "VM-entry with non-launched executing VMCS.",
541	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
542	/* 19 */ "VMCALL with non-clear VMCS.",
543	/* 20 */ "VMCALL with invalid VM-exit control fields.",
544	/* 21 */ "(Not Used)",
545	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
546	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
547	/* 24 */ "VMCALL with invalid SMM-monitor features.",
548	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
549	/* 26 */ "VM-entry with events blocked by MOV SS.",
550	/* 27 */ "(Not Used)",
551	/* 28 */ "Invalid operand to INVEPT/INVVPID."
552	};
553	#endif /* VBOX_STRICT */
554
555
556
557	/**
558	* Updates the VM's last error record.
559	*
560	* If there was a VMX instruction error, reads the error data from the VMCS and
561	* updates VCPU's last error record as well.
562	*
563	* @param pVM The cross context VM structure.
564	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
565	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
566	* VERR_VMX_INVALID_VMCS_FIELD.
567	* @param rc The error code.
568	*/
569	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
570	{
571	AssertPtr(pVM);
572	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
573	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
574	{
575	AssertPtrReturnVoid(pVCpu);
576	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
577	}
578	pVM->hm.s.lLastError = rc;
579	}
580
581
582	/**
583	* Reads the VM-entry interruption-information field from the VMCS into the VMX
584	* transient structure.
585	*
586	* @returns VBox status code.
587	* @param pVmxTransient Pointer to the VMX transient structure.
588	*
589	* @remarks No-long-jump zone!!!
590	*/
591	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
592	{
593	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
594	AssertRCReturn(rc, rc);
595	return VINF_SUCCESS;
596	}
597
598
599	#ifdef VBOX_STRICT
600	/**
601	* Reads the VM-entry exception error code field from the VMCS into
602	* the VMX transient structure.
603	*
604	* @returns VBox status code.
605	* @param pVmxTransient Pointer to the VMX transient structure.
606	*
607	* @remarks No-long-jump zone!!!
608	*/
609	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
610	{
611	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
612	AssertRCReturn(rc, rc);
613	return VINF_SUCCESS;
614	}
615	#endif /* VBOX_STRICT */
616
617
618	#ifdef VBOX_STRICT
619	/**
620	* Reads the VM-entry exception error code field from the VMCS into
621	* the VMX transient structure.
622	*
623	* @returns VBox status code.
624	* @param pVmxTransient Pointer to the VMX transient structure.
625	*
626	* @remarks No-long-jump zone!!!
627	*/
628	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
629	{
630	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
631	AssertRCReturn(rc, rc);
632	return VINF_SUCCESS;
633	}
634	#endif /* VBOX_STRICT */
635
636
637	/**
638	* Reads the VM-exit interruption-information field from the VMCS into the VMX
639	* transient structure.
640	*
641	* @returns VBox status code.
642	* @param pVmxTransient Pointer to the VMX transient structure.
643	*/
644	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
645	{
646	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
647	{
648	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
649	AssertRCReturn(rc, rc);
650	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
651	}
652	return VINF_SUCCESS;
653	}
654
655
656	/**
657	* Reads the VM-exit interruption error code from the VMCS into the VMX
658	* transient structure.
659	*
660	* @returns VBox status code.
661	* @param pVmxTransient Pointer to the VMX transient structure.
662	*/
663	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
664	{
665	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
666	{
667	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
668	AssertRCReturn(rc, rc);
669	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
670	}
671	return VINF_SUCCESS;
672	}
673
674
675	/**
676	* Reads the VM-exit instruction length field from the VMCS into the VMX
677	* transient structure.
678	*
679	* @returns VBox status code.
680	* @param pVmxTransient Pointer to the VMX transient structure.
681	*/
682	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
683	{
684	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
685	{
686	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
687	AssertRCReturn(rc, rc);
688	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
689	}
690	return VINF_SUCCESS;
691	}
692
693
694	/**
695	* Reads the VM-exit instruction-information field from the VMCS into
696	* the VMX transient structure.
697	*
698	* @returns VBox status code.
699	* @param pVmxTransient Pointer to the VMX transient structure.
700	*/
701	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
702	{
703	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
704	{
705	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
706	AssertRCReturn(rc, rc);
707	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
708	}
709	return VINF_SUCCESS;
710	}
711
712
713	/**
714	* Reads the exit code qualification from the VMCS into the VMX transient
715	* structure.
716	*
717	* @returns VBox status code.
718	* @param pVCpu The cross context virtual CPU structure of the
719	* calling EMT. (Required for the VMCS cache case.)
720	* @param pVmxTransient Pointer to the VMX transient structure.
721	*/
722	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
723	{
724	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
725	{
726	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
727	AssertRCReturn(rc, rc);
728	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
729	}
730	return VINF_SUCCESS;
731	}
732
733
734	/**
735	* Reads the IDT-vectoring information field from the VMCS into the VMX
736	* transient structure.
737	*
738	* @returns VBox status code.
739	* @param pVmxTransient Pointer to the VMX transient structure.
740	*
741	* @remarks No-long-jump zone!!!
742	*/
743	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
744	{
745	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
746	{
747	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
748	AssertRCReturn(rc, rc);
749	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
750	}
751	return VINF_SUCCESS;
752	}
753
754
755	/**
756	* Reads the IDT-vectoring error code from the VMCS into the VMX
757	* transient structure.
758	*
759	* @returns VBox status code.
760	* @param pVmxTransient Pointer to the VMX transient structure.
761	*/
762	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
763	{
764	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
765	{
766	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
767	AssertRCReturn(rc, rc);
768	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
769	}
770	return VINF_SUCCESS;
771	}
772
773
774	/**
775	* Enters VMX root mode operation on the current CPU.
776	*
777	* @returns VBox status code.
778	* @param pVM The cross context VM structure. Can be
779	* NULL, after a resume.
780	* @param HCPhysCpuPage Physical address of the VMXON region.
781	* @param pvCpuPage Pointer to the VMXON region.
782	*/
783	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
784	{
785	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
786	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
787	Assert(pvCpuPage);
788	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
789
790	if (pVM)
791	{
792	/* Write the VMCS revision dword to the VMXON region. */
793	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
794	}
795
796	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
797	RTCCUINTREG fEFlags = ASMIntDisableFlags();
798
799	/* Enable the VMX bit in CR4 if necessary. */
800	RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
801
802	/* Enter VMX root mode. */
803	int rc = VMXEnable(HCPhysCpuPage);
804	if (RT_FAILURE(rc))
805	{
806	if (!(uOldCr4 & X86_CR4_VMXE))
807	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
808
809	if (pVM)
810	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
811	}
812
813	/* Restore interrupts. */
814	ASMSetFlags(fEFlags);
815	return rc;
816	}
817
818
819	/**
820	* Exits VMX root mode operation on the current CPU.
821	*
822	* @returns VBox status code.
823	*/
824	static int hmR0VmxLeaveRootMode(void)
825	{
826	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
827
828	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
829	RTCCUINTREG fEFlags = ASMIntDisableFlags();
830
831	/* If we're for some reason not in VMX root mode, then don't leave it. */
832	RTCCUINTREG uHostCR4 = ASMGetCR4();
833
834	int rc;
835	if (uHostCR4 & X86_CR4_VMXE)
836	{
837	/* Exit VMX root mode and clear the VMX bit in CR4. */
838	VMXDisable();
839	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
840	rc = VINF_SUCCESS;
841	}
842	else
843	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
844
845	/* Restore interrupts. */
846	ASMSetFlags(fEFlags);
847	return rc;
848	}
849
850
851	/**
852	* Allocates and maps one physically contiguous page. The allocated page is
853	* zero'd out. (Used by various VT-x structures).
854	*
855	* @returns IPRT status code.
856	* @param pMemObj Pointer to the ring-0 memory object.
857	* @param ppVirt Where to store the virtual address of the
858	* allocation.
859	* @param pHCPhys Where to store the physical address of the
860	* allocation.
861	*/
862	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
863	{
864	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
865	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
866	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
867
868	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
869	if (RT_FAILURE(rc))
870	return rc;
871	ppVirt = RTR0MemObjAddress(pMemObj);
872	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
873	ASMMemZero32(*ppVirt, PAGE_SIZE);
874	return VINF_SUCCESS;
875	}
876
877
878	/**
879	* Frees and unmaps an allocated physical page.
880	*
881	* @param pMemObj Pointer to the ring-0 memory object.
882	* @param ppVirt Where to re-initialize the virtual address of
883	* allocation as 0.
884	* @param pHCPhys Where to re-initialize the physical address of the
885	* allocation as 0.
886	*/
887	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
888	{
889	AssertPtr(pMemObj);
890	AssertPtr(ppVirt);
891	AssertPtr(pHCPhys);
892	if (*pMemObj != NIL_RTR0MEMOBJ)
893	{
894	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
895	AssertRC(rc);
896	*pMemObj = NIL_RTR0MEMOBJ;
897	*ppVirt = 0;
898	*pHCPhys = 0;
899	}
900	}
901
902
903	/**
904	* Worker function to free VT-x related structures.
905	*
906	* @returns IPRT status code.
907	* @param pVM The cross context VM structure.
908	*/
909	static void hmR0VmxStructsFree(PVM pVM)
910	{
911	for (VMCPUID i = 0; i < pVM->cCpus; i++)
912	{
913	PVMCPU pVCpu = &pVM->aCpus[i];
914	AssertPtr(pVCpu);
915
916	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
917	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
918
919	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
920	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
921
922	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
923	}
924
925	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
926	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
927	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
928	#endif
929	}
930
931
932	/**
933	* Worker function to allocate VT-x related VM structures.
934	*
935	* @returns IPRT status code.
936	* @param pVM The cross context VM structure.
937	*/
938	static int hmR0VmxStructsAlloc(PVM pVM)
939	{
940	/*
941	* Initialize members up-front so we can cleanup properly on allocation failure.
942	*/
943	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
944	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
945	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
946	pVM->hm.s.vmx.HCPhys##a_Name = 0;
947
948	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
949	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
950	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
951	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
952
953	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
954	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
955	#endif
956	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
957
958	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
959	for (VMCPUID i = 0; i < pVM->cCpus; i++)
960	{
961	PVMCPU pVCpu = &pVM->aCpus[i];
962	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
963	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
964	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
965	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
966	}
967	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
968	#undef VMXLOCAL_INIT_VM_MEMOBJ
969
970	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
971	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
972	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
973	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
974
975	/*
976	* Allocate all the VT-x structures.
977	*/
978	int rc = VINF_SUCCESS;
979	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
980	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
981	if (RT_FAILURE(rc))
982	goto cleanup;
983	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
984	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
985	#endif
986
987	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
988	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
989	{
990	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
991	&pVM->hm.s.vmx.HCPhysApicAccess);
992	if (RT_FAILURE(rc))
993	goto cleanup;
994	}
995
996	/*
997	* Initialize per-VCPU VT-x structures.
998	*/
999	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1000	{
1001	PVMCPU pVCpu = &pVM->aCpus[i];
1002	AssertPtr(pVCpu);
1003
1004	/* Allocate the VM control structure (VMCS). */
1005	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1006	if (RT_FAILURE(rc))
1007	goto cleanup;
1008
1009	/* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1010	if ( PDMHasApic(pVM)
1011	&& (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW))
1012	{
1013	rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1014	NULL /* pR3Ptr /, NULL / pRCPtr */);
1015	if (RT_FAILURE(rc))
1016	goto cleanup;
1017	}
1018
1019	/*
1020	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1021	* transparent accesses of specific MSRs.
1022	*
1023	* If the condition for enabling MSR bitmaps changes here, don't forget to
1024	* update HMAreMsrBitmapsAvailable().
1025	*/
1026	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1027	{
1028	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1029	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1030	if (RT_FAILURE(rc))
1031	goto cleanup;
1032	ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1033	}
1034
1035	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1036	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1037	if (RT_FAILURE(rc))
1038	goto cleanup;
1039
1040	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1041	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1042	if (RT_FAILURE(rc))
1043	goto cleanup;
1044	}
1045
1046	return VINF_SUCCESS;
1047
1048	cleanup:
1049	hmR0VmxStructsFree(pVM);
1050	return rc;
1051	}
1052
1053
1054	/**
1055	* Does global VT-x initialization (called during module initialization).
1056	*
1057	* @returns VBox status code.
1058	*/
1059	VMMR0DECL(int) VMXR0GlobalInit(void)
1060	{
1061	#ifdef HMVMX_USE_FUNCTION_TABLE
1062	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1063	# ifdef VBOX_STRICT
1064	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1065	Assert(g_apfnVMExitHandlers[i]);
1066	# endif
1067	#endif
1068	return VINF_SUCCESS;
1069	}
1070
1071
1072	/**
1073	* Does global VT-x termination (called during module termination).
1074	*/
1075	VMMR0DECL(void) VMXR0GlobalTerm()
1076	{
1077	/* Nothing to do currently. */
1078	}
1079
1080
1081	/**
1082	* Sets up and activates VT-x on the current CPU.
1083	*
1084	* @returns VBox status code.
1085	* @param pCpu Pointer to the global CPU info struct.
1086	* @param pVM The cross context VM structure. Can be
1087	* NULL after a host resume operation.
1088	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1089	* fEnabledByHost is @c true).
1090	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1091	* @a fEnabledByHost is @c true).
1092	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1093	* enable VT-x on the host.
1094	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1095	*/
1096	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1097	void *pvMsrs)
1098	{
1099	Assert(pCpu);
1100	Assert(pvMsrs);
1101	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1102
1103	/* Enable VT-x if it's not already enabled by the host. */
1104	if (!fEnabledByHost)
1105	{
1106	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1107	if (RT_FAILURE(rc))
1108	return rc;
1109	}
1110
1111	/*
1112	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1113	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1114	*/
1115	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1116	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1117	{
1118	hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1119	pCpu->fFlushAsidBeforeUse = false;
1120	}
1121	else
1122	pCpu->fFlushAsidBeforeUse = true;
1123
1124	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1125	++pCpu->cTlbFlushes;
1126
1127	return VINF_SUCCESS;
1128	}
1129
1130
1131	/**
1132	* Deactivates VT-x on the current CPU.
1133	*
1134	* @returns VBox status code.
1135	* @param pCpu Pointer to the global CPU info struct.
1136	* @param pvCpuPage Pointer to the VMXON region.
1137	* @param HCPhysCpuPage Physical address of the VMXON region.
1138	*
1139	* @remarks This function should never be called when SUPR0EnableVTx() or
1140	* similar was used to enable VT-x on the host.
1141	*/
1142	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1143	{
1144	NOREF(pCpu);
1145	NOREF(pvCpuPage);
1146	NOREF(HCPhysCpuPage);
1147
1148	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1149	return hmR0VmxLeaveRootMode();
1150	}
1151
1152
1153	/**
1154	* Sets the permission bits for the specified MSR in the MSR bitmap.
1155	*
1156	* @param pVCpu The cross context virtual CPU structure.
1157	* @param uMsr The MSR value.
1158	* @param enmRead Whether reading this MSR causes a VM-exit.
1159	* @param enmWrite Whether writing this MSR causes a VM-exit.
1160	*/
1161	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1162	{
1163	int32_t iBit;
1164	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1165
1166	/*
1167	* Layout:
1168	* 0x000 - 0x3ff - Low MSR read bits
1169	* 0x400 - 0x7ff - High MSR read bits
1170	* 0x800 - 0xbff - Low MSR write bits
1171	* 0xc00 - 0xfff - High MSR write bits
1172	*/
1173	if (uMsr <= 0x00001FFF)
1174	iBit = uMsr;
1175	else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1176	{
1177	iBit = uMsr - UINT32_C(0xC0000000);
1178	pbMsrBitmap += 0x400;
1179	}
1180	else
1181	AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1182
1183	Assert(iBit <= 0x1fff);
1184	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1185	ASMBitSet(pbMsrBitmap, iBit);
1186	else
1187	ASMBitClear(pbMsrBitmap, iBit);
1188
1189	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1190	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1191	else
1192	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1193	}
1194
1195
1196	#ifdef VBOX_STRICT
1197	/**
1198	* Gets the permission bits for the specified MSR in the MSR bitmap.
1199	*
1200	* @returns VBox status code.
1201	* @retval VINF_SUCCESS if the specified MSR is found.
1202	* @retval VERR_NOT_FOUND if the specified MSR is not found.
1203	* @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1204	*
1205	* @param pVCpu The cross context virtual CPU structure.
1206	* @param uMsr The MSR.
1207	* @param penmRead Where to store the read permissions.
1208	* @param penmWrite Where to store the write permissions.
1209	*/
1210	static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1211	{
1212	AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1213	AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1214	int32_t iBit;
1215	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1216
1217	/* See hmR0VmxSetMsrPermission() for the layout. */
1218	if (uMsr <= 0x00001FFF)
1219	iBit = uMsr;
1220	else if ( uMsr >= 0xC0000000
1221	&& uMsr <= 0xC0001FFF)
1222	{
1223	iBit = (uMsr - 0xC0000000);
1224	pbMsrBitmap += 0x400;
1225	}
1226	else
1227	AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1228
1229	Assert(iBit <= 0x1fff);
1230	if (ASMBitTest(pbMsrBitmap, iBit))
1231	*penmRead = VMXMSREXIT_INTERCEPT_READ;
1232	else
1233	*penmRead = VMXMSREXIT_PASSTHRU_READ;
1234
1235	if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1236	*penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1237	else
1238	*penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1239	return VINF_SUCCESS;
1240	}
1241	#endif /* VBOX_STRICT */
1242
1243
1244	/**
1245	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1246	* area.
1247	*
1248	* @returns VBox status code.
1249	* @param pVCpu The cross context virtual CPU structure.
1250	* @param cMsrs The number of MSRs.
1251	*/
1252	DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1253	{
1254	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1255	uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1256	if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1257	{
1258	LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1259	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1260	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1261	}
1262
1263	/* Update number of guest MSRs to load/store across the world-switch. */
1264	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1265	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1266
1267	/* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1268	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1269	AssertRCReturn(rc, rc);
1270
1271	/* Update the VCPU's copy of the MSR count. */
1272	pVCpu->hm.s.vmx.cMsrs = cMsrs;
1273
1274	return VINF_SUCCESS;
1275	}
1276
1277
1278	/**
1279	* Adds a new (or updates the value of an existing) guest/host MSR
1280	* pair to be swapped during the world-switch as part of the
1281	* auto-load/store MSR area in the VMCS.
1282	*
1283	* @returns VBox status code.
1284	* @param pVCpu The cross context virtual CPU structure.
1285	* @param uMsr The MSR.
1286	* @param uGuestMsrValue Value of the guest MSR.
1287	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1288	* necessary.
1289	* @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1290	* its value was updated. Optional, can be NULL.
1291	*/
1292	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1293	bool *pfAddedAndUpdated)
1294	{
1295	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1296	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1297	uint32_t i;
1298	for (i = 0; i < cMsrs; i++)
1299	{
1300	if (pGuestMsr->u32Msr == uMsr)
1301	break;
1302	pGuestMsr++;
1303	}
1304
1305	bool fAdded = false;
1306	if (i == cMsrs)
1307	{
1308	++cMsrs;
1309	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1310	AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1311
1312	/* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1313	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1314	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1315
1316	fAdded = true;
1317	}
1318
1319	/* Update the MSR values in the auto-load/store MSR area. */
1320	pGuestMsr->u32Msr = uMsr;
1321	pGuestMsr->u64Value = uGuestMsrValue;
1322
1323	/* Create/update the MSR slot in the host MSR area. */
1324	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1325	pHostMsr += i;
1326	pHostMsr->u32Msr = uMsr;
1327
1328	/*
1329	* Update the host MSR only when requested by the caller AND when we're
1330	* adding it to the auto-load/store area. Otherwise, it would have been
1331	* updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1332	*/
1333	bool fUpdatedMsrValue = false;
1334	if ( fAdded
1335	&& fUpdateHostMsr)
1336	{
1337	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1338	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1339	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1340	fUpdatedMsrValue = true;
1341	}
1342
1343	if (pfAddedAndUpdated)
1344	*pfAddedAndUpdated = fUpdatedMsrValue;
1345	return VINF_SUCCESS;
1346	}
1347
1348
1349	/**
1350	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1351	* auto-load/store MSR area in the VMCS.
1352	*
1353	* @returns VBox status code.
1354	* @param pVCpu The cross context virtual CPU structure.
1355	* @param uMsr The MSR.
1356	*/
1357	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1358	{
1359	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1360	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1361	for (uint32_t i = 0; i < cMsrs; i++)
1362	{
1363	/* Find the MSR. */
1364	if (pGuestMsr->u32Msr == uMsr)
1365	{
1366	/* If it's the last MSR, simply reduce the count. */
1367	if (i == cMsrs - 1)
1368	{
1369	--cMsrs;
1370	break;
1371	}
1372
1373	/* Remove it by swapping the last MSR in place of it, and reducing the count. */
1374	PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1375	pLastGuestMsr += cMsrs - 1;
1376	pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1377	pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1378
1379	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1380	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1381	pLastHostMsr += cMsrs - 1;
1382	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1383	pHostMsr->u64Value = pLastHostMsr->u64Value;
1384	--cMsrs;
1385	break;
1386	}
1387	pGuestMsr++;
1388	}
1389
1390	/* Update the VMCS if the count changed (meaning the MSR was found). */
1391	if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1392	{
1393	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1394	AssertRCReturn(rc, rc);
1395
1396	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1397	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1398	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1399
1400	Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1401	return VINF_SUCCESS;
1402	}
1403
1404	return VERR_NOT_FOUND;
1405	}
1406
1407
1408	/**
1409	* Checks if the specified guest MSR is part of the auto-load/store area in
1410	* the VMCS.
1411	*
1412	* @returns true if found, false otherwise.
1413	* @param pVCpu The cross context virtual CPU structure.
1414	* @param uMsr The MSR to find.
1415	*/
1416	static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1417	{
1418	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1419	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1420
1421	for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1422	{
1423	if (pGuestMsr->u32Msr == uMsr)
1424	return true;
1425	}
1426	return false;
1427	}
1428
1429
1430	/**
1431	* Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1432	*
1433	* @param pVCpu The cross context virtual CPU structure.
1434	*
1435	* @remarks No-long-jump zone!!!
1436	*/
1437	static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1438	{
1439	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1440	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1441	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1442	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1443
1444	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1445	{
1446	AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1447
1448	/*
1449	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1450	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1451	*/
1452	if (pHostMsr->u32Msr == MSR_K6_EFER)
1453	pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1454	else
1455	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1456	}
1457
1458	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1459	}
1460
1461
1462	/**
1463	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1464	* perform lazy restoration of the host MSRs while leaving VT-x.
1465	*
1466	* @param pVCpu The cross context virtual CPU structure.
1467	*
1468	* @remarks No-long-jump zone!!!
1469	*/
1470	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1471	{
1472	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1473
1474	/*
1475	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1476	*/
1477	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1478	{
1479	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1480	#if HC_ARCH_BITS == 64
1481	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1482	{
1483	pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1484	pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1485	pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1486	pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1487	}
1488	#endif
1489	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1490	}
1491	}
1492
1493
1494	/**
1495	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1496	* lazily while leaving VT-x.
1497	*
1498	* @returns true if it does, false otherwise.
1499	* @param pVCpu The cross context virtual CPU structure.
1500	* @param uMsr The MSR to check.
1501	*/
1502	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1503	{
1504	NOREF(pVCpu);
1505	#if HC_ARCH_BITS == 64
1506	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1507	{
1508	switch (uMsr)
1509	{
1510	case MSR_K8_LSTAR:
1511	case MSR_K6_STAR:
1512	case MSR_K8_SF_MASK:
1513	case MSR_K8_KERNEL_GS_BASE:
1514	return true;
1515	}
1516	}
1517	#else
1518	RT_NOREF(pVCpu, uMsr);
1519	#endif
1520	return false;
1521	}
1522
1523
1524	/**
1525	* Saves a set of guest MSRs back into the guest-CPU context.
1526	*
1527	* @param pVCpu The cross context virtual CPU structure.
1528	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1529	* out-of-sync. Make sure to update the required fields
1530	* before using them.
1531	*
1532	* @remarks No-long-jump zone!!!
1533	*/
1534	static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1535	{
1536	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1537	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1538
1539	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1540	{
1541	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1542	#if HC_ARCH_BITS == 64
1543	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1544	{
1545	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1546	pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1547	pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1548	pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1549	}
1550	#else
1551	NOREF(pMixedCtx);
1552	#endif
1553	}
1554	}
1555
1556
1557	/**
1558	* Loads a set of guests MSRs to allow read/passthru to the guest.
1559	*
1560	* The name of this function is slightly confusing. This function does NOT
1561	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1562	* common prefix for functions dealing with "lazy restoration" of the shared
1563	* MSRs.
1564	*
1565	* @param pVCpu The cross context virtual CPU structure.
1566	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1567	* out-of-sync. Make sure to update the required fields
1568	* before using them.
1569	*
1570	* @remarks No-long-jump zone!!!
1571	*/
1572	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1573	{
1574	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1575	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1576
1577	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1578	#if HC_ARCH_BITS == 64
1579	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1580	{
1581	/*
1582	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1583	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1584	* we can skip a few MSR writes.
1585	*
1586	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1587	* guest MSR values in the guest-CPU context might be different to what's currently
1588	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
1589	* CPU, see @bugref{8728}.
1590	*/
1591	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1592	&& pMixedCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1593	&& pMixedCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1594	&& pMixedCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1595	&& pMixedCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1596	{
1597	#ifdef VBOX_STRICT
1598	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pMixedCtx->msrKERNELGSBASE);
1599	Assert(ASMRdMsr(MSR_K8_LSTAR) == pMixedCtx->msrLSTAR);
1600	Assert(ASMRdMsr(MSR_K6_STAR) == pMixedCtx->msrSTAR);
1601	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pMixedCtx->msrSFMASK);
1602	#endif
1603	}
1604	else
1605	{
1606	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1607	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1608	ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1609	ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1610	}
1611	}
1612	#else
1613	RT_NOREF(pMixedCtx);
1614	#endif
1615	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
1616	}
1617
1618
1619	/**
1620	* Performs lazy restoration of the set of host MSRs if they were previously
1621	* loaded with guest MSR values.
1622	*
1623	* @param pVCpu The cross context virtual CPU structure.
1624	*
1625	* @remarks No-long-jump zone!!!
1626	* @remarks The guest MSRs should have been saved back into the guest-CPU
1627	* context by hmR0VmxSaveGuestLazyMsrs()!!!
1628	*/
1629	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1630	{
1631	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1632	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1633
1634	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1635	{
1636	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1637	#if HC_ARCH_BITS == 64
1638	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1639	{
1640	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1641	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1642	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1643	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1644	}
1645	#endif
1646	}
1647	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
1648	}
1649
1650
1651	/**
1652	* Verifies that our cached values of the VMCS controls are all
1653	* consistent with what's actually present in the VMCS.
1654	*
1655	* @returns VBox status code.
1656	* @param pVCpu The cross context virtual CPU structure.
1657	*/
1658	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1659	{
1660	uint32_t u32Val;
1661	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1662	AssertRCReturn(rc, rc);
1663	AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1664	VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1665
1666	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1667	AssertRCReturn(rc, rc);
1668	AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1669	VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1670
1671	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1672	AssertRCReturn(rc, rc);
1673	AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1674	VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1675
1676	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1677	AssertRCReturn(rc, rc);
1678	AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1679	VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1680
1681	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1682	{
1683	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1684	AssertRCReturn(rc, rc);
1685	AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1686	("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1687	VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1688	}
1689
1690	return VINF_SUCCESS;
1691	}
1692
1693
1694	#ifdef VBOX_STRICT
1695	/**
1696	* Verifies that our cached host EFER value has not changed
1697	* since we cached it.
1698	*
1699	* @param pVCpu The cross context virtual CPU structure.
1700	*/
1701	static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1702	{
1703	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1704
1705	if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1706	{
1707	uint64_t u64Val;
1708	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1709	AssertRC(rc);
1710
1711	uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1712	AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1713	}
1714	}
1715
1716
1717	/**
1718	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1719	* VMCS are correct.
1720	*
1721	* @param pVCpu The cross context virtual CPU structure.
1722	*/
1723	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1724	{
1725	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1726
1727	/* Verify MSR counts in the VMCS are what we think it should be. */
1728	uint32_t cMsrs;
1729	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1730	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1731
1732	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1733	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1734
1735	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1736	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1737
1738	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1739	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1740	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1741	{
1742	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1743	AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1744	pGuestMsr->u32Msr, cMsrs));
1745
1746	uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1747	AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1748	pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1749
1750	/* Verify that the permissions are as expected in the MSR bitmap. */
1751	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1752	{
1753	VMXMSREXITREAD enmRead;
1754	VMXMSREXITWRITE enmWrite;
1755	rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1756	AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1757	if (pGuestMsr->u32Msr == MSR_K6_EFER)
1758	{
1759	AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1760	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1761	}
1762	else
1763	{
1764	AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1765	pGuestMsr->u32Msr, cMsrs));
1766	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1767	pGuestMsr->u32Msr, cMsrs));
1768	}
1769	}
1770	}
1771	}
1772	#endif /* VBOX_STRICT */
1773
1774
1775	/**
1776	* Flushes the TLB using EPT.
1777	*
1778	* @returns VBox status code.
1779	* @param pVCpu The cross context virtual CPU structure of the calling
1780	* EMT. Can be NULL depending on @a enmFlush.
1781	* @param enmFlush Type of flush.
1782	*
1783	* @remarks Caller is responsible for making sure this function is called only
1784	* when NestedPaging is supported and providing @a enmFlush that is
1785	* supported by the CPU.
1786	* @remarks Can be called with interrupts disabled.
1787	*/
1788	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1789	{
1790	uint64_t au64Descriptor[2];
1791	if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1792	au64Descriptor[0] = 0;
1793	else
1794	{
1795	Assert(pVCpu);
1796	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1797	}
1798	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1799
1800	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1801	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1802	rc));
1803	if ( RT_SUCCESS(rc)
1804	&& pVCpu)
1805	{
1806	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1807	}
1808	}
1809
1810
1811	/**
1812	* Flushes the TLB using VPID.
1813	*
1814	* @returns VBox status code.
1815	* @param pVM The cross context VM structure.
1816	* @param pVCpu The cross context virtual CPU structure of the calling
1817	* EMT. Can be NULL depending on @a enmFlush.
1818	* @param enmFlush Type of flush.
1819	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1820	* on @a enmFlush).
1821	*
1822	* @remarks Can be called with interrupts disabled.
1823	*/
1824	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1825	{
1826	NOREF(pVM);
1827	AssertPtr(pVM);
1828	Assert(pVM->hm.s.vmx.fVpid);
1829
1830	uint64_t au64Descriptor[2];
1831	if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1832	{
1833	au64Descriptor[0] = 0;
1834	au64Descriptor[1] = 0;
1835	}
1836	else
1837	{
1838	AssertPtr(pVCpu);
1839	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1840	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1841	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1842	au64Descriptor[1] = GCPtr;
1843	}
1844
1845	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1846	AssertMsg(rc == VINF_SUCCESS,
1847	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1848	if ( RT_SUCCESS(rc)
1849	&& pVCpu)
1850	{
1851	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1852	}
1853	}
1854
1855
1856	/**
1857	* Invalidates a guest page by guest virtual address. Only relevant for
1858	* EPT/VPID, otherwise there is nothing really to invalidate.
1859	*
1860	* @returns VBox status code.
1861	* @param pVM The cross context VM structure.
1862	* @param pVCpu The cross context virtual CPU structure.
1863	* @param GCVirt Guest virtual address of the page to invalidate.
1864	*/
1865	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1866	{
1867	AssertPtr(pVM);
1868	AssertPtr(pVCpu);
1869	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1870
1871	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1872	if (!fFlushPending)
1873	{
1874	/*
1875	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1876	* See @bugref{6043} and @bugref{6177}.
1877	*
1878	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1879	* function maybe called in a loop with individual addresses.
1880	*/
1881	if (pVM->hm.s.vmx.fVpid)
1882	{
1883	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1884	{
1885	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1886	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1887	}
1888	else
1889	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1890	}
1891	else if (pVM->hm.s.fNestedPaging)
1892	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1893	}
1894
1895	return VINF_SUCCESS;
1896	}
1897
1898
1899	/**
1900	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1901	* otherwise there is nothing really to invalidate.
1902	*
1903	* @returns VBox status code.
1904	* @param pVM The cross context VM structure.
1905	* @param pVCpu The cross context virtual CPU structure.
1906	* @param GCPhys Guest physical address of the page to invalidate.
1907	*/
1908	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1909	{
1910	NOREF(pVM); NOREF(GCPhys);
1911	LogFlowFunc(("%RGp\n", GCPhys));
1912
1913	/*
1914	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1915	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1916	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1917	*/
1918	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1919	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1920	return VINF_SUCCESS;
1921	}
1922
1923
1924	/**
1925	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1926	* case where neither EPT nor VPID is supported by the CPU.
1927	*
1928	* @param pVM The cross context VM structure.
1929	* @param pVCpu The cross context virtual CPU structure.
1930	* @param pCpu Pointer to the global HM struct.
1931	*
1932	* @remarks Called with interrupts disabled.
1933	*/
1934	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1935	{
1936	AssertPtr(pVCpu);
1937	AssertPtr(pCpu);
1938	NOREF(pVM);
1939
1940	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1941
1942	Assert(pCpu->idCpu != NIL_RTCPUID);
1943	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1944	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1945	pVCpu->hm.s.fForceTLBFlush = false;
1946	return;
1947	}
1948
1949
1950	/**
1951	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1952	*
1953	* @param pVM The cross context VM structure.
1954	* @param pVCpu The cross context virtual CPU structure.
1955	* @param pCpu Pointer to the global HM CPU struct.
1956	* @remarks All references to "ASID" in this function pertains to "VPID" in
1957	* Intel's nomenclature. The reason is, to avoid confusion in compare
1958	* statements since the host-CPU copies are named "ASID".
1959	*
1960	* @remarks Called with interrupts disabled.
1961	*/
1962	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1963	{
1964	#ifdef VBOX_WITH_STATISTICS
1965	bool fTlbFlushed = false;
1966	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1967	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1968	if (!fTlbFlushed) \
1969	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1970	} while (0)
1971	#else
1972	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1973	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1974	#endif
1975
1976	AssertPtr(pVM);
1977	AssertPtr(pCpu);
1978	AssertPtr(pVCpu);
1979	Assert(pCpu->idCpu != NIL_RTCPUID);
1980
1981	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1982	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1983	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1984
1985	/*
1986	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1987	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1988	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1989	*/
1990	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1991	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1992	{
1993	++pCpu->uCurrentAsid;
1994	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1995	{
1996	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1997	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1998	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1999	}
2000
2001	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2002	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2003	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2004
2005	/*
2006	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2007	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2008	*/
2009	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2010	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2011	HMVMX_SET_TAGGED_TLB_FLUSHED();
2012	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
2013	}
2014
2015	/* Check for explicit TLB flushes. */
2016	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2017	{
2018	/*
2019	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
2020	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2021	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2022	* but not guest-physical mappings.
2023	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2024	*/
2025	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2026	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2027	HMVMX_SET_TAGGED_TLB_FLUSHED();
2028	}
2029
2030	pVCpu->hm.s.fForceTLBFlush = false;
2031	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2032
2033	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2034	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2035	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2036	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2037	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2038	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2039	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2040	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2041	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2042
2043	/* Update VMCS with the VPID. */
2044	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2045	AssertRC(rc);
2046
2047	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2048	}
2049
2050
2051	/**
2052	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2053	*
2054	* @returns VBox status code.
2055	* @param pVM The cross context VM structure.
2056	* @param pVCpu The cross context virtual CPU structure.
2057	* @param pCpu Pointer to the global HM CPU struct.
2058	*
2059	* @remarks Called with interrupts disabled.
2060	*/
2061	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2062	{
2063	AssertPtr(pVM);
2064	AssertPtr(pVCpu);
2065	AssertPtr(pCpu);
2066	Assert(pCpu->idCpu != NIL_RTCPUID);
2067	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2068	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2069
2070	/*
2071	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2072	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2073	*/
2074	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2075	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2076	{
2077	pVCpu->hm.s.fForceTLBFlush = true;
2078	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2079	}
2080
2081	/* Check for explicit TLB flushes. */
2082	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2083	{
2084	pVCpu->hm.s.fForceTLBFlush = true;
2085	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2086	}
2087
2088	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2089	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2090
2091	if (pVCpu->hm.s.fForceTLBFlush)
2092	{
2093	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2094	pVCpu->hm.s.fForceTLBFlush = false;
2095	}
2096	}
2097
2098
2099	/**
2100	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2101	*
2102	* @returns VBox status code.
2103	* @param pVM The cross context VM structure.
2104	* @param pVCpu The cross context virtual CPU structure.
2105	* @param pCpu Pointer to the global HM CPU struct.
2106	*
2107	* @remarks Called with interrupts disabled.
2108	*/
2109	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2110	{
2111	AssertPtr(pVM);
2112	AssertPtr(pVCpu);
2113	AssertPtr(pCpu);
2114	Assert(pCpu->idCpu != NIL_RTCPUID);
2115	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2116	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2117
2118	/*
2119	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2120	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2121	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2122	*/
2123	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2124	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2125	{
2126	pVCpu->hm.s.fForceTLBFlush = true;
2127	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2128	}
2129
2130	/* Check for explicit TLB flushes. */
2131	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2132	{
2133	/*
2134	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2135	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2136	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2137	*/
2138	pVCpu->hm.s.fForceTLBFlush = true;
2139	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2140	}
2141
2142	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2143	if (pVCpu->hm.s.fForceTLBFlush)
2144	{
2145	++pCpu->uCurrentAsid;
2146	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2147	{
2148	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2149	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2150	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2151	}
2152
2153	pVCpu->hm.s.fForceTLBFlush = false;
2154	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2155	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2156	if (pCpu->fFlushAsidBeforeUse)
2157	{
2158	if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2159	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2160	else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2161	{
2162	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2163	pCpu->fFlushAsidBeforeUse = false;
2164	}
2165	else
2166	{
2167	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2168	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2169	}
2170	}
2171	}
2172
2173	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2174	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2175	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2176	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2177	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2178	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2179	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2180
2181	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2182	AssertRC(rc);
2183	}
2184
2185
2186	/**
2187	* Flushes the guest TLB entry based on CPU capabilities.
2188	*
2189	* @param pVCpu The cross context virtual CPU structure.
2190	* @param pCpu Pointer to the global HM CPU struct.
2191	*/
2192	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2193	{
2194	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2195	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2196	#endif
2197	PVM pVM = pVCpu->CTX_SUFF(pVM);
2198	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2199	{
2200	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2201	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2202	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2203	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2204	default:
2205	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2206	break;
2207	}
2208
2209	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2210	}
2211
2212
2213	/**
2214	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2215	* TLB entries from the host TLB before VM-entry.
2216	*
2217	* @returns VBox status code.
2218	* @param pVM The cross context VM structure.
2219	*/
2220	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2221	{
2222	/*
2223	* Determine optimal flush type for Nested Paging.
2224	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2225	* guest execution (see hmR3InitFinalizeR0()).
2226	*/
2227	if (pVM->hm.s.fNestedPaging)
2228	{
2229	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2230	{
2231	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2232	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2233	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2234	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2235	else
2236	{
2237	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2238	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2239	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2240	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2241	}
2242
2243	/* Make sure the write-back cacheable memory type for EPT is supported. */
2244	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2245	{
2246	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2247	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2248	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2249	}
2250
2251	/* EPT requires a page-walk length of 4. */
2252	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2253	{
2254	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2255	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2256	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2257	}
2258	}
2259	else
2260	{
2261	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2262	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2263	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2264	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2265	}
2266	}
2267
2268	/*
2269	* Determine optimal flush type for VPID.
2270	*/
2271	if (pVM->hm.s.vmx.fVpid)
2272	{
2273	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2274	{
2275	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2276	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2277	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2278	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2279	else
2280	{
2281	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2282	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2283	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2284	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2285	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2286	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2287	pVM->hm.s.vmx.fVpid = false;
2288	}
2289	}
2290	else
2291	{
2292	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2293	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2294	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2295	pVM->hm.s.vmx.fVpid = false;
2296	}
2297	}
2298
2299	/*
2300	* Setup the handler for flushing tagged-TLBs.
2301	*/
2302	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2303	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2304	else if (pVM->hm.s.fNestedPaging)
2305	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2306	else if (pVM->hm.s.vmx.fVpid)
2307	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2308	else
2309	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2310	return VINF_SUCCESS;
2311	}
2312
2313
2314	/**
2315	* Sets up pin-based VM-execution controls in the VMCS.
2316	*
2317	* @returns VBox status code.
2318	* @param pVM The cross context VM structure.
2319	* @param pVCpu The cross context virtual CPU structure.
2320	*/
2321	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2322	{
2323	AssertPtr(pVM);
2324	AssertPtr(pVCpu);
2325
2326	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2327	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2328
2329	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2330	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2331
2332	if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2333	val \|= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2334
2335	/* Enable the VMX preemption timer. */
2336	if (pVM->hm.s.vmx.fUsePreemptTimer)
2337	{
2338	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2339	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2340	}
2341
2342	#if 0
2343	/* Enable posted-interrupt processing. */
2344	if (pVM->hm.s.fPostedIntrs)
2345	{
2346	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2347	Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2348	val \|= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2349	}
2350	#endif
2351
2352	if ((val & zap) != val)
2353	{
2354	LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2355	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2356	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2357	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2358	}
2359
2360	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2361	AssertRCReturn(rc, rc);
2362
2363	pVCpu->hm.s.vmx.u32PinCtls = val;
2364	return rc;
2365	}
2366
2367
2368	/**
2369	* Sets up processor-based VM-execution controls in the VMCS.
2370	*
2371	* @returns VBox status code.
2372	* @param pVM The cross context VM structure.
2373	* @param pVCpu The cross context virtual CPU structure.
2374	*/
2375	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2376	{
2377	AssertPtr(pVM);
2378	AssertPtr(pVCpu);
2379
2380	int rc = VERR_INTERNAL_ERROR_5;
2381	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2382	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2383
2384	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2385	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2386	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2387	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2388	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2389	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2390	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2391
2392	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2393	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2394	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2395	{
2396	LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2397	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2398	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2399	}
2400
2401	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2402	if (!pVM->hm.s.fNestedPaging)
2403	{
2404	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2405	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2406	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2407	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2408	}
2409
2410	/* Use TPR shadowing if supported by the CPU. */
2411	if ( PDMHasApic(pVM)
2412	&& pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2413	{
2414	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2415	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2416	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2417	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2418	AssertRCReturn(rc, rc);
2419
2420	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2421	/* CR8 writes cause a VM-exit based on TPR threshold. */
2422	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2423	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2424	}
2425	else
2426	{
2427	/*
2428	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2429	* Set this control only for 64-bit guests.
2430	*/
2431	if (pVM->hm.s.fAllow64BitGuests)
2432	{
2433	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2434	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2435	}
2436	}
2437
2438	/* Use MSR-bitmaps if supported by the CPU. */
2439	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2440	{
2441	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2442
2443	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2444	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2445	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2446	AssertRCReturn(rc, rc);
2447
2448	/*
2449	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2450	* automatically using dedicated fields in the VMCS.
2451	*/
2452	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2453	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2454	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2455	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2456	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2457
2458	#if HC_ARCH_BITS == 64
2459	/*
2460	* Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2461	*/
2462	if (pVM->hm.s.fAllow64BitGuests)
2463	{
2464	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2465	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2466	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2467	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2468	}
2469	#endif
2470	/* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2471	}
2472
2473	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2474	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2475	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2476
2477	if ((val & zap) != val)
2478	{
2479	LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2480	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2481	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2482	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2483	}
2484
2485	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2486	AssertRCReturn(rc, rc);
2487
2488	pVCpu->hm.s.vmx.u32ProcCtls = val;
2489
2490	/*
2491	* Secondary processor-based VM-execution controls.
2492	*/
2493	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2494	{
2495	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2496	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2497
2498	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2499	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2500
2501	if (pVM->hm.s.fNestedPaging)
2502	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2503	else
2504	{
2505	/*
2506	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2507	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2508	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2509	*/
2510	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2511	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2512	}
2513
2514	if (pVM->hm.s.vmx.fVpid)
2515	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2516
2517	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2518	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2519
2520	#if 0
2521	if (pVM->hm.s.fVirtApicRegs)
2522	{
2523	Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2524	val \|= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2525
2526	Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2527	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2528	}
2529	#endif
2530
2531	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2532	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2533	* done dynamically. */
2534	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2535	{
2536	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2537	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2538	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2539	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2540	AssertRCReturn(rc, rc);
2541	}
2542
2543	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2544	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2545
2546	if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2547	&& pVM->hm.s.vmx.cPleGapTicks
2548	&& pVM->hm.s.vmx.cPleWindowTicks)
2549	{
2550	val \|= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2551
2552	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2553	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2554	AssertRCReturn(rc, rc);
2555	}
2556
2557	if ((val & zap) != val)
2558	{
2559	LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2560	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2561	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2562	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2563	}
2564
2565	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2566	AssertRCReturn(rc, rc);
2567
2568	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2569	}
2570	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2571	{
2572	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2573	"available\n"));
2574	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2575	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2576	}
2577
2578	return VINF_SUCCESS;
2579	}
2580
2581
2582	/**
2583	* Sets up miscellaneous (everything other than Pin & Processor-based
2584	* VM-execution) control fields in the VMCS.
2585	*
2586	* @returns VBox status code.
2587	* @param pVM The cross context VM structure.
2588	* @param pVCpu The cross context virtual CPU structure.
2589	*/
2590	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2591	{
2592	NOREF(pVM);
2593	AssertPtr(pVM);
2594	AssertPtr(pVCpu);
2595
2596	int rc = VERR_GENERAL_FAILURE;
2597
2598	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2599	#if 0
2600	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2601	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2602	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2603
2604	/*
2605	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2606	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2607	* We thus use the exception bitmap to control it rather than use both.
2608	*/
2609	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2610	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2611
2612	/** @todo Explore possibility of using IO-bitmaps. */
2613	/* All IO & IOIO instructions cause VM-exits. */
2614	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2615	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2616
2617	/* Initialize the MSR-bitmap area. */
2618	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2619	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2620	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2621	AssertRCReturn(rc, rc);
2622	#endif
2623
2624	/* Setup MSR auto-load/store area. */
2625	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2626	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2627	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2628	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2629	AssertRCReturn(rc, rc);
2630
2631	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2632	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2633	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2634	AssertRCReturn(rc, rc);
2635
2636	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2637	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2638	AssertRCReturn(rc, rc);
2639
2640	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2641	#if 0
2642	/* Setup debug controls */
2643	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2644	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2645	AssertRCReturn(rc, rc);
2646	#endif
2647
2648	return rc;
2649	}
2650
2651
2652	/**
2653	* Sets up the initial exception bitmap in the VMCS based on static conditions.
2654	*
2655	* We shall setup those exception intercepts that don't change during the
2656	* lifetime of the VM here. The rest are done dynamically while loading the
2657	* guest state.
2658	*
2659	* @returns VBox status code.
2660	* @param pVM The cross context VM structure.
2661	* @param pVCpu The cross context virtual CPU structure.
2662	*/
2663	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2664	{
2665	AssertPtr(pVM);
2666	AssertPtr(pVCpu);
2667
2668	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2669
2670	uint32_t u32XcptBitmap = 0;
2671
2672	/* Must always intercept #AC to prevent the guest from hanging the CPU. */
2673	u32XcptBitmap \|= RT_BIT_32(X86_XCPT_AC);
2674
2675	/* Because we need to maintain the DR6 state even when intercepting DRx reads
2676	and writes, and because recursive #DBs can cause the CPU hang, we must always
2677	intercept #DB. */
2678	u32XcptBitmap \|= RT_BIT_32(X86_XCPT_DB);
2679
2680	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2681	if (!pVM->hm.s.fNestedPaging)
2682	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2683
2684	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2685	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2686	AssertRCReturn(rc, rc);
2687	return rc;
2688	}
2689
2690
2691	/**
2692	* Sets up the initial guest-state mask. The guest-state mask is consulted
2693	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2694	* for the nested virtualization case (as it would cause a VM-exit).
2695	*
2696	* @param pVCpu The cross context virtual CPU structure.
2697	*/
2698	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2699	{
2700	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2701	HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2702	return VINF_SUCCESS;
2703	}
2704
2705
2706	/**
2707	* Does per-VM VT-x initialization.
2708	*
2709	* @returns VBox status code.
2710	* @param pVM The cross context VM structure.
2711	*/
2712	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2713	{
2714	LogFlowFunc(("pVM=%p\n", pVM));
2715
2716	int rc = hmR0VmxStructsAlloc(pVM);
2717	if (RT_FAILURE(rc))
2718	{
2719	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2720	return rc;
2721	}
2722
2723	return VINF_SUCCESS;
2724	}
2725
2726
2727	/**
2728	* Does per-VM VT-x termination.
2729	*
2730	* @returns VBox status code.
2731	* @param pVM The cross context VM structure.
2732	*/
2733	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2734	{
2735	LogFlowFunc(("pVM=%p\n", pVM));
2736
2737	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2738	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2739	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2740	#endif
2741	hmR0VmxStructsFree(pVM);
2742	return VINF_SUCCESS;
2743	}
2744
2745
2746	/**
2747	* Sets up the VM for execution under VT-x.
2748	* This function is only called once per-VM during initialization.
2749	*
2750	* @returns VBox status code.
2751	* @param pVM The cross context VM structure.
2752	*/
2753	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2754	{
2755	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2756	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2757
2758	LogFlowFunc(("pVM=%p\n", pVM));
2759
2760	/*
2761	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2762	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2763	*/
2764	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2765	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2766	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2767	{
2768	LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2769	return VERR_INTERNAL_ERROR;
2770	}
2771
2772	/* Initialize these always, see hmR3InitFinalizeR0().*/
2773	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2774	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2775
2776	/* Setup the tagged-TLB flush handlers. */
2777	int rc = hmR0VmxSetupTaggedTlb(pVM);
2778	if (RT_FAILURE(rc))
2779	{
2780	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2781	return rc;
2782	}
2783
2784	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
2785	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2786	#if HC_ARCH_BITS == 64
2787	if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2788	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2789	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2790	{
2791	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2792	}
2793	#endif
2794
2795	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2796	RTCCUINTREG uHostCR4 = ASMGetCR4();
2797	if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2798	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2799
2800	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2801	{
2802	PVMCPU pVCpu = &pVM->aCpus[i];
2803	AssertPtr(pVCpu);
2804	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2805
2806	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2807	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2808
2809	/* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2810	Assert(!pVCpu->hm.s.idxExitHistoryFree);
2811	HMCPU_EXIT_HISTORY_RESET(pVCpu);
2812
2813	/* Set revision dword at the beginning of the VMCS structure. */
2814	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2815
2816	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2817	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2818	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2819	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2820
2821	/* Load this VMCS as the current VMCS. */
2822	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2823	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2824	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2825
2826	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2827	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2828	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2829
2830	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2831	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2832	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2833
2834	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2835	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2836	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2837
2838	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2839	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2840	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2841
2842	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2843	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2844	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2845
2846	#if HC_ARCH_BITS == 32
2847	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2848	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2849	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2850	#endif
2851
2852	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2853	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2854	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2855	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2856
2857	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2858
2859	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2860	}
2861
2862	return VINF_SUCCESS;
2863	}
2864
2865
2866	/**
2867	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2868	* the VMCS.
2869	*
2870	* @returns VBox status code.
2871	* @param pVM The cross context VM structure.
2872	* @param pVCpu The cross context virtual CPU structure.
2873	*/
2874	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2875	{
2876	NOREF(pVM); NOREF(pVCpu);
2877
2878	RTCCUINTREG uReg = ASMGetCR0();
2879	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2880	AssertRCReturn(rc, rc);
2881
2882	uReg = ASMGetCR3();
2883	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2884	AssertRCReturn(rc, rc);
2885
2886	uReg = ASMGetCR4();
2887	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2888	AssertRCReturn(rc, rc);
2889	return rc;
2890	}
2891
2892
2893	#if HC_ARCH_BITS == 64
2894	/**
2895	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2896	* requirements. See hmR0VmxSaveHostSegmentRegs().
2897	*/
2898	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2899	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2900	{ \
2901	bool fValidSelector = true; \
2902	if ((selValue) & X86_SEL_LDT) \
2903	{ \
2904	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2905	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2906	} \
2907	if (fValidSelector) \
2908	{ \
2909	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2910	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2911	} \
2912	(selValue) = 0; \
2913	}
2914	#endif
2915
2916
2917	/**
2918	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2919	* the host-state area in the VMCS.
2920	*
2921	* @returns VBox status code.
2922	* @param pVM The cross context VM structure.
2923	* @param pVCpu The cross context virtual CPU structure.
2924	*/
2925	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2926	{
2927	int rc = VERR_INTERNAL_ERROR_5;
2928
2929	#if HC_ARCH_BITS == 64
2930	/*
2931	* If we've executed guest code using VT-x, the host-state bits will be messed up. We
2932	* should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2933	*
2934	* This apparently can happen (most likely the FPU changes), deal with it rather than asserting.
2935	* Was observed booting Solaris10u10 32-bit guest.
2936	*/
2937	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2938	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2939	{
2940	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2941	pVCpu->idCpu));
2942	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2943	}
2944	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2945	#else
2946	RT_NOREF(pVCpu);
2947	#endif
2948
2949	/*
2950	* Host DS, ES, FS and GS segment registers.
2951	*/
2952	#if HC_ARCH_BITS == 64
2953	RTSEL uSelDS = ASMGetDS();
2954	RTSEL uSelES = ASMGetES();
2955	RTSEL uSelFS = ASMGetFS();
2956	RTSEL uSelGS = ASMGetGS();
2957	#else
2958	RTSEL uSelDS = 0;
2959	RTSEL uSelES = 0;
2960	RTSEL uSelFS = 0;
2961	RTSEL uSelGS = 0;
2962	#endif
2963
2964	/*
2965	* Host CS and SS segment registers.
2966	*/
2967	RTSEL uSelCS = ASMGetCS();
2968	RTSEL uSelSS = ASMGetSS();
2969
2970	/*
2971	* Host TR segment register.
2972	*/
2973	RTSEL uSelTR = ASMGetTR();
2974
2975	#if HC_ARCH_BITS == 64
2976	/*
2977	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2978	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2979	*/
2980	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2981	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2982	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2983	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2984	# undef VMXLOCAL_ADJUST_HOST_SEG
2985	#endif
2986
2987	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2988	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2989	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2990	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2991	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2992	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2993	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2994	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2995	Assert(uSelCS);
2996	Assert(uSelTR);
2997
2998	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2999	#if 0
3000	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
3001	Assert(uSelSS != 0);
3002	#endif
3003
3004	/* Write these host selector fields into the host-state area in the VMCS. */
3005	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3006	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3007	#if HC_ARCH_BITS == 64
3008	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3009	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3010	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3011	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3012	#else
3013	NOREF(uSelDS);
3014	NOREF(uSelES);
3015	NOREF(uSelFS);
3016	NOREF(uSelGS);
3017	#endif
3018	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3019	AssertRCReturn(rc, rc);
3020
3021	/*
3022	* Host GDTR and IDTR.
3023	*/
3024	RTGDTR Gdtr;
3025	RTIDTR Idtr;
3026	RT_ZERO(Gdtr);
3027	RT_ZERO(Idtr);
3028	ASMGetGDTR(&Gdtr);
3029	ASMGetIDTR(&Idtr);
3030	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3031	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3032	AssertRCReturn(rc, rc);
3033
3034	#if HC_ARCH_BITS == 64
3035	/*
3036	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3037	* maximum limit (0xffff) on every VM-exit.
3038	*/
3039	if (Gdtr.cbGdt != 0xffff)
3040	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
3041
3042	/*
3043	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3044	* and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3045	* bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3046	* on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3047	* but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3048	* hosts where we are pretty sure it won't cause trouble.
3049	*/
3050	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
3051	if (Idtr.cbIdt < 0x0fff)
3052	# else
3053	if (Idtr.cbIdt != 0xffff)
3054	# endif
3055	{
3056	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
3057	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3058	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3059	}
3060	#endif
3061
3062	/*
3063	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3064	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3065	*/
3066	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3067	("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3068	VERR_VMX_INVALID_HOST_STATE);
3069
3070	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3071	#if HC_ARCH_BITS == 64
3072	uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3073
3074	/*
3075	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3076	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3077	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3078	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3079	*
3080	* [1] See Intel spec. 3.5 "System Descriptor Types".
3081	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3082	*/
3083	Assert(pDesc->System.u4Type == 11);
3084	if ( pDesc->System.u16LimitLow != 0x67
3085	\|\| pDesc->System.u4LimitHigh)
3086	{
3087	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
3088	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3089	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3090	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
3091	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3092	}
3093
3094	/*
3095	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3096	*/
3097	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
3098	{
3099	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3100	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3101	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3102	{
3103	/* The GDT is read-only but the writable GDT is available. */
3104	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3105	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3106	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3107	AssertRCReturn(rc, rc);
3108	}
3109	}
3110	#else
3111	NOREF(pVM);
3112	uintptr_t uTRBase = X86DESC_BASE(pDesc);
3113	#endif
3114	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3115	AssertRCReturn(rc, rc);
3116
3117	/*
3118	* Host FS base and GS base.
3119	*/
3120	#if HC_ARCH_BITS == 64
3121	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3122	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3123	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3124	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3125	AssertRCReturn(rc, rc);
3126
3127	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3128	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3129	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3130	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3131	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3132	#endif
3133	return rc;
3134	}
3135
3136
3137	/**
3138	* Saves certain host MSRs in the VM-exit MSR-load area and some in the
3139	* host-state area of the VMCS. Theses MSRs will be automatically restored on
3140	* the host after every successful VM-exit.
3141	*
3142	* @returns VBox status code.
3143	* @param pVM The cross context VM structure.
3144	* @param pVCpu The cross context virtual CPU structure.
3145	*
3146	* @remarks No-long-jump zone!!!
3147	*/
3148	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3149	{
3150	NOREF(pVM);
3151
3152	AssertPtr(pVCpu);
3153	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3154
3155	/*
3156	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3157	* rather than swapping them on every VM-entry.
3158	*/
3159	hmR0VmxLazySaveHostMsrs(pVCpu);
3160
3161	/*
3162	* Host Sysenter MSRs.
3163	*/
3164	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3165	#if HC_ARCH_BITS == 32
3166	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3167	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3168	#else
3169	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3170	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3171	#endif
3172	AssertRCReturn(rc, rc);
3173
3174	/*
3175	* Host EFER MSR.
3176	* If the CPU supports the newer VMCS controls for managing EFER, use it.
3177	* Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3178	*/
3179	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3180	{
3181	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3182	AssertRCReturn(rc, rc);
3183	}
3184
3185	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3186	* hmR0VmxLoadGuestExitCtls() !! */
3187
3188	return rc;
3189	}
3190
3191
3192	/**
3193	* Figures out if we need to swap the EFER MSR which is particularly expensive.
3194	*
3195	* We check all relevant bits. For now, that's everything besides LMA/LME, as
3196	* these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3197	* hmR0VMxLoadGuestEntryCtls().
3198	*
3199	* @returns true if we need to load guest EFER, false otherwise.
3200	* @param pVCpu The cross context virtual CPU structure.
3201	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3202	* out-of-sync. Make sure to update the required fields
3203	* before using them.
3204	*
3205	* @remarks Requires EFER, CR4.
3206	* @remarks No-long-jump zone!!!
3207	*/
3208	static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3209	{
3210	#ifdef HMVMX_ALWAYS_SWAP_EFER
3211	return true;
3212	#endif
3213
3214	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3215	/* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3216	if (CPUMIsGuestInLongMode(pVCpu))
3217	return false;
3218	#endif
3219
3220	PVM pVM = pVCpu->CTX_SUFF(pVM);
3221	uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3222	uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3223
3224	/*
3225	* For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3226	* guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3227	*/
3228	if ( CPUMIsGuestInLongMode(pVCpu)
3229	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3230	{
3231	return true;
3232	}
3233
3234	/*
3235	* If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3236	* affects guest paging. 64-bit paging implies CR4.PAE as well.
3237	* See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3238	*/
3239	if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3240	&& (pMixedCtx->cr0 & X86_CR0_PG)
3241	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3242	{
3243	/* Assert that host is PAE capable. */
3244	Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3245	return true;
3246	}
3247
3248	/** @todo Check the latest Intel spec. for any other bits,
3249	* like SMEP/SMAP? */
3250	return false;
3251	}
3252
3253
3254	/**
3255	* Sets up VM-entry controls in the VMCS. These controls can affect things done
3256	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3257	* controls".
3258	*
3259	* @returns VBox status code.
3260	* @param pVCpu The cross context virtual CPU structure.
3261	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3262	* out-of-sync. Make sure to update the required fields
3263	* before using them.
3264	*
3265	* @remarks Requires EFER.
3266	* @remarks No-long-jump zone!!!
3267	*/
3268	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3269	{
3270	int rc = VINF_SUCCESS;
3271	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3272	{
3273	PVM pVM = pVCpu->CTX_SUFF(pVM);
3274	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3275	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3276
3277	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3278	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3279
3280	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3281	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3282	{
3283	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3284	Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3285	}
3286	else
3287	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3288
3289	/* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3290	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3291	&& hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3292	{
3293	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3294	Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3295	}
3296
3297	/*
3298	* The following should -not- be set (since we're not in SMM mode):
3299	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3300	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3301	*/
3302
3303	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3304	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3305
3306	if ((val & zap) != val)
3307	{
3308	LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3309	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3310	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3311	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3312	}
3313
3314	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3315	AssertRCReturn(rc, rc);
3316
3317	pVCpu->hm.s.vmx.u32EntryCtls = val;
3318	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3319	}
3320	return rc;
3321	}
3322
3323
3324	/**
3325	* Sets up the VM-exit controls in the VMCS.
3326	*
3327	* @returns VBox status code.
3328	* @param pVCpu The cross context virtual CPU structure.
3329	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3330	* out-of-sync. Make sure to update the required fields
3331	* before using them.
3332	*
3333	* @remarks Requires EFER.
3334	*/
3335	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3336	{
3337	NOREF(pMixedCtx);
3338
3339	int rc = VINF_SUCCESS;
3340	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3341	{
3342	PVM pVM = pVCpu->CTX_SUFF(pVM);
3343	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3344	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3345
3346	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3347	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3348
3349	/*
3350	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3351	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3352	*/
3353	#if HC_ARCH_BITS == 64
3354	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3355	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3356	#else
3357	Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3358	\|\| pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3359	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3360	if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3361	{
3362	/* The switcher returns to long mode, EFER is managed by the switcher. */
3363	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3364	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3365	}
3366	else
3367	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3368	#endif
3369
3370	/* If the newer VMCS fields for managing EFER exists, use it. */
3371	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3372	&& hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3373	{
3374	val \|= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3375	\| VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3376	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3377	}
3378
3379	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3380	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3381
3382	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3383	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3384	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3385
3386	if ( pVM->hm.s.vmx.fUsePreemptTimer
3387	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3388	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3389
3390	if ((val & zap) != val)
3391	{
3392	LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3393	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3394	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3395	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3396	}
3397
3398	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3399	AssertRCReturn(rc, rc);
3400
3401	pVCpu->hm.s.vmx.u32ExitCtls = val;
3402	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3403	}
3404	return rc;
3405	}
3406
3407
3408	/**
3409	* Sets the TPR threshold in the VMCS.
3410	*
3411	* @returns VBox status code.
3412	* @param pVCpu The cross context virtual CPU structure.
3413	* @param u32TprThreshold The TPR threshold (task-priority class only).
3414	*/
3415	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3416	{
3417	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3418	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3419	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3420	}
3421
3422
3423	/**
3424	* Loads the guest APIC and related state.
3425	*
3426	* @returns VBox status code.
3427	* @param pVCpu The cross context virtual CPU structure.
3428	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3429	* out-of-sync. Make sure to update the required fields
3430	* before using them.
3431	*
3432	* @remarks No-long-jump zone!!!
3433	*/
3434	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3435	{
3436	NOREF(pMixedCtx);
3437
3438	int rc = VINF_SUCCESS;
3439	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3440	{
3441	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3442	&& APICIsEnabled(pVCpu))
3443	{
3444	/*
3445	* Setup TPR shadowing.
3446	*/
3447	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3448	{
3449	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3450
3451	bool fPendingIntr = false;
3452	uint8_t u8Tpr = 0;
3453	uint8_t u8PendingIntr = 0;
3454	rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3455	AssertRCReturn(rc, rc);
3456
3457	/*
3458	* If there are interrupts pending but masked by the TPR, instruct VT-x to cause a TPR-below-threshold VM-exit
3459	* when the guest lowers its TPR below the priority of the pending interrupt so we can deliver the interrupt.
3460	* If there are no interrupts pending, set threshold to 0 to not cause any TPR-below-threshold VM-exits.
3461	*/
3462	pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3463	uint32_t u32TprThreshold = 0;
3464	if (fPendingIntr)
3465	{
3466	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3467	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3468	const uint8_t u8TprPriority = u8Tpr >> 4;
3469	if (u8PendingPriority <= u8TprPriority)
3470	u32TprThreshold = u8PendingPriority;
3471	}
3472
3473	rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3474	AssertRCReturn(rc, rc);
3475	}
3476	}
3477	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3478	}
3479
3480	return rc;
3481	}
3482
3483
3484	/**
3485	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3486	*
3487	* @returns Guest's interruptibility-state.
3488	* @param pVCpu The cross context virtual CPU structure.
3489	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3490	* out-of-sync. Make sure to update the required fields
3491	* before using them.
3492	*
3493	* @remarks No-long-jump zone!!!
3494	*/
3495	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3496	{
3497	/*
3498	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3499	*/
3500	uint32_t uIntrState = 0;
3501	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3502	{
3503	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3504	AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS),
3505	("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3506	if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3507	{
3508	if (pMixedCtx->eflags.Bits.u1IF)
3509	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3510	else
3511	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3512	}
3513	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3514	{
3515	/*
3516	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3517	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3518	*/
3519	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3520	}
3521	}
3522
3523	/*
3524	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3525	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3526	* setting this would block host-NMIs and IRET will not clear the blocking.
3527	*
3528	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3529	*/
3530	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3531	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3532	{
3533	uIntrState \|= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3534	}
3535
3536	return uIntrState;
3537	}
3538
3539
3540	/**
3541	* Loads the guest's interruptibility-state into the guest-state area in the
3542	* VMCS.
3543	*
3544	* @returns VBox status code.
3545	* @param pVCpu The cross context virtual CPU structure.
3546	* @param uIntrState The interruptibility-state to set.
3547	*/
3548	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3549	{
3550	NOREF(pVCpu);
3551	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3552	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3553	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3554	AssertRC(rc);
3555	return rc;
3556	}
3557
3558
3559	/**
3560	* Loads the exception intercepts required for guest execution in the VMCS.
3561	*
3562	* @returns VBox status code.
3563	* @param pVCpu The cross context virtual CPU structure.
3564	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3565	* out-of-sync. Make sure to update the required fields
3566	* before using them.
3567	*/
3568	static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3569	{
3570	NOREF(pMixedCtx);
3571	int rc = VINF_SUCCESS;
3572	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3573	{
3574	/* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3575	if (pVCpu->hm.s.fGIMTrapXcptUD)
3576	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_UD);
3577	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3578	else
3579	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3580	#endif
3581
3582	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3583	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3584
3585	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3586	AssertRCReturn(rc, rc);
3587
3588	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3589	Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3590	pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3591	}
3592	return rc;
3593	}
3594
3595
3596	/**
3597	* Loads the guest's RIP into the guest-state area in the VMCS.
3598	*
3599	* @returns VBox status code.
3600	* @param pVCpu The cross context virtual CPU structure.
3601	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3602	* out-of-sync. Make sure to update the required fields
3603	* before using them.
3604	*
3605	* @remarks No-long-jump zone!!!
3606	*/
3607	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3608	{
3609	int rc = VINF_SUCCESS;
3610	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3611	{
3612	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3613	AssertRCReturn(rc, rc);
3614
3615	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3616	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3617	HMCPU_CF_VALUE(pVCpu)));
3618	}
3619	return rc;
3620	}
3621
3622
3623	/**
3624	* Loads the guest's RSP into the guest-state area in the VMCS.
3625	*
3626	* @returns VBox status code.
3627	* @param pVCpu The cross context virtual CPU structure.
3628	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3629	* out-of-sync. Make sure to update the required fields
3630	* before using them.
3631	*
3632	* @remarks No-long-jump zone!!!
3633	*/
3634	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3635	{
3636	int rc = VINF_SUCCESS;
3637	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3638	{
3639	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3640	AssertRCReturn(rc, rc);
3641
3642	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3643	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3644	}
3645	return rc;
3646	}
3647
3648
3649	/**
3650	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
3651	*
3652	* @returns VBox status code.
3653	* @param pVCpu The cross context virtual CPU structure.
3654	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3655	* out-of-sync. Make sure to update the required fields
3656	* before using them.
3657	*
3658	* @remarks No-long-jump zone!!!
3659	*/
3660	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3661	{
3662	int rc = VINF_SUCCESS;
3663	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3664	{
3665	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3666	Let us assert it as such and use 32-bit VMWRITE. */
3667	Assert(!(pMixedCtx->rflags.u64 >> 32));
3668	X86EFLAGS Eflags = pMixedCtx->eflags;
3669	/** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3670	* shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3671	* These will never be cleared/set, unless some other part of the VMM
3672	* code is buggy - in which case we're better of finding and fixing
3673	* those bugs than hiding them. */
3674	Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3675	Assert(!(Eflags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
3676	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3677	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3678
3679	/*
3680	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3681	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3682	*/
3683	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3684	{
3685	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3686	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3687	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3688	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3689	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3690	}
3691
3692	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3693	AssertRCReturn(rc, rc);
3694
3695	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3696	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3697	}
3698	return rc;
3699	}
3700
3701
3702	/**
3703	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3704	*
3705	* @returns VBox status code.
3706	* @param pVCpu The cross context virtual CPU structure.
3707	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3708	* out-of-sync. Make sure to update the required fields
3709	* before using them.
3710	*
3711	* @remarks No-long-jump zone!!!
3712	*/
3713	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3714	{
3715	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3716	rc \|= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3717	rc \|= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3718	AssertRCReturn(rc, rc);
3719	return rc;
3720	}
3721
3722
3723	/**
3724	* Loads the guest CR0 control register into the guest-state area in the VMCS.
3725	* CR0 is partially shared with the host and we have to consider the FPU bits.
3726	*
3727	* @returns VBox status code.
3728	* @param pVCpu The cross context virtual CPU structure.
3729	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3730	* out-of-sync. Make sure to update the required fields
3731	* before using them.
3732	*
3733	* @remarks No-long-jump zone!!!
3734	*/
3735	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3736	{
3737	/*
3738	* Guest CR0.
3739	* Guest FPU.
3740	*/
3741	int rc = VINF_SUCCESS;
3742	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3743	{
3744	Assert(!(pMixedCtx->cr0 >> 32));
3745	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3746	PVM pVM = pVCpu->CTX_SUFF(pVM);
3747
3748	/* The guest's view (read access) of its CR0 is unblemished. */
3749	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3750	AssertRCReturn(rc, rc);
3751	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3752
3753	/* Setup VT-x's view of the guest CR0. */
3754	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3755	if (pVM->hm.s.fNestedPaging)
3756	{
3757	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3758	{
3759	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3760	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3761	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3762	}
3763	else
3764	{
3765	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3766	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3767	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3768	}
3769
3770	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3771	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3772	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3773
3774	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3775	AssertRCReturn(rc, rc);
3776	}
3777	else
3778	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3779
3780	/*
3781	* Guest FPU bits.
3782	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3783	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3784	*/
3785	u32GuestCR0 \|= X86_CR0_NE;
3786	bool fInterceptNM = false;
3787	if (CPUMIsGuestFPUStateActive(pVCpu))
3788	{
3789	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3790	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3791	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3792	}
3793	else
3794	{
3795	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3796	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3797	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3798	}
3799
3800	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3801	bool fInterceptMF = false;
3802	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3803	fInterceptMF = true;
3804
3805	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3806	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3807	{
3808	Assert(PDMVmmDevHeapIsEnabled(pVM));
3809	Assert(pVM->hm.s.vmx.pRealModeTSS);
3810	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3811	fInterceptNM = true;
3812	fInterceptMF = true;
3813	}
3814	else
3815	{
3816	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3817	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3818	}
3819	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3820
3821	if (fInterceptNM)
3822	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3823	else
3824	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3825
3826	if (fInterceptMF)
3827	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3828	else
3829	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3830
3831	/* Additional intercepts for debugging, define these yourself explicitly. */
3832	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3833	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3834	\| RT_BIT(X86_XCPT_BP)
3835	\| RT_BIT(X86_XCPT_DE)
3836	\| RT_BIT(X86_XCPT_NM)
3837	\| RT_BIT(X86_XCPT_TS)
3838	\| RT_BIT(X86_XCPT_UD)
3839	\| RT_BIT(X86_XCPT_NP)
3840	\| RT_BIT(X86_XCPT_SS)
3841	\| RT_BIT(X86_XCPT_GP)
3842	\| RT_BIT(X86_XCPT_PF)
3843	\| RT_BIT(X86_XCPT_MF)
3844	;
3845	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3846	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3847	#endif
3848
3849	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3850
3851	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3852	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3853	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3854	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3855	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3856	else
3857	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3858
3859	u32GuestCR0 \|= uSetCR0;
3860	u32GuestCR0 &= uZapCR0;
3861	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3862
3863	/* Write VT-x's view of the guest CR0 into the VMCS. */
3864	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3865	AssertRCReturn(rc, rc);
3866	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3867	uZapCR0));
3868
3869	/*
3870	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3871	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3872	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3873	*/
3874	uint32_t u32CR0Mask = 0;
3875	u32CR0Mask = X86_CR0_PE
3876	\| X86_CR0_NE
3877	\| X86_CR0_WP
3878	\| X86_CR0_PG
3879	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3880	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3881	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3882
3883	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3884	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3885	* and @bugref{6944}. */
3886	#if 0
3887	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3888	u32CR0Mask &= ~X86_CR0_PE;
3889	#endif
3890	if (pVM->hm.s.fNestedPaging)
3891	u32CR0Mask &= ~X86_CR0_WP;
3892
3893	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3894	if (fInterceptNM)
3895	{
3896	u32CR0Mask \|= X86_CR0_TS
3897	\| X86_CR0_MP;
3898	}
3899
3900	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3901	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3902	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3903	AssertRCReturn(rc, rc);
3904	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3905
3906	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3907	}
3908	return rc;
3909	}
3910
3911
3912	/**
3913	* Loads the guest control registers (CR3, CR4) into the guest-state area
3914	* in the VMCS.
3915	*
3916	* @returns VBox strict status code.
3917	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3918	* without unrestricted guest access and the VMMDev is not presently
3919	* mapped (e.g. EFI32).
3920	*
3921	* @param pVCpu The cross context virtual CPU structure.
3922	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3923	* out-of-sync. Make sure to update the required fields
3924	* before using them.
3925	*
3926	* @remarks No-long-jump zone!!!
3927	*/
3928	static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3929	{
3930	int rc = VINF_SUCCESS;
3931	PVM pVM = pVCpu->CTX_SUFF(pVM);
3932
3933	/*
3934	* Guest CR2.
3935	* It's always loaded in the assembler code. Nothing to do here.
3936	*/
3937
3938	/*
3939	* Guest CR3.
3940	*/
3941	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3942	{
3943	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3944	if (pVM->hm.s.fNestedPaging)
3945	{
3946	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3947
3948	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3949	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3950	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3951	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3952
3953	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3954	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3955	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3956
3957	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3958	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3959	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3960	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3961	AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3962	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3963	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3964
3965	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3966	AssertRCReturn(rc, rc);
3967	Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3968
3969	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3970	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3971	{
3972	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3973	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3974	{
3975	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3976	AssertRCReturn(rc, rc);
3977	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3978	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3979	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3980	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3981	AssertRCReturn(rc, rc);
3982	}
3983
3984	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3985	have Unrestricted Execution to handle the guest when it's not using paging. */
3986	GCPhysGuestCR3 = pMixedCtx->cr3;
3987	}
3988	else
3989	{
3990	/*
3991	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3992	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3993	* EPT takes care of translating it to host-physical addresses.
3994	*/
3995	RTGCPHYS GCPhys;
3996	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3997
3998	/* We obtain it here every time as the guest could have relocated this PCI region. */
3999	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
4000	if (RT_SUCCESS(rc))
4001	{ /* likely */ }
4002	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
4003	{
4004	Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
4005	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
4006	}
4007	else
4008	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
4009
4010	GCPhysGuestCR3 = GCPhys;
4011	}
4012
4013	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
4014	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4015	}
4016	else
4017	{
4018	/* Non-nested paging case, just use the hypervisor's CR3. */
4019	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4020
4021	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
4022	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4023	}
4024	AssertRCReturn(rc, rc);
4025
4026	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
4027	}
4028
4029	/*
4030	* Guest CR4.
4031	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
4032	*/
4033	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
4034	{
4035	Assert(!(pMixedCtx->cr4 >> 32));
4036	uint32_t u32GuestCR4 = pMixedCtx->cr4;
4037
4038	/* The guest's view of its CR4 is unblemished. */
4039	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
4040	AssertRCReturn(rc, rc);
4041	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
4042
4043	/* Setup VT-x's view of the guest CR4. */
4044	/*
4045	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4046	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4047	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4048	*/
4049	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4050	{
4051	Assert(pVM->hm.s.vmx.pRealModeTSS);
4052	Assert(PDMVmmDevHeapIsEnabled(pVM));
4053	u32GuestCR4 &= ~X86_CR4_VME;
4054	}
4055
4056	if (pVM->hm.s.fNestedPaging)
4057	{
4058	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4059	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
4060	{
4061	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4062	u32GuestCR4 \|= X86_CR4_PSE;
4063	/* Our identity mapping is a 32-bit page directory. */
4064	u32GuestCR4 &= ~X86_CR4_PAE;
4065	}
4066	/* else use guest CR4.*/
4067	}
4068	else
4069	{
4070	/*
4071	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4072	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4073	*/
4074	switch (pVCpu->hm.s.enmShadowMode)
4075	{
4076	case PGMMODE_REAL: /* Real-mode. */
4077	case PGMMODE_PROTECTED: /* Protected mode without paging. */
4078	case PGMMODE_32_BIT: /* 32-bit paging. */
4079	{
4080	u32GuestCR4 &= ~X86_CR4_PAE;
4081	break;
4082	}
4083
4084	case PGMMODE_PAE: /* PAE paging. */
4085	case PGMMODE_PAE_NX: /* PAE paging with NX. */
4086	{
4087	u32GuestCR4 \|= X86_CR4_PAE;
4088	break;
4089	}
4090
4091	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4092	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4093	#ifdef VBOX_ENABLE_64_BITS_GUESTS
4094	break;
4095	#endif
4096	default:
4097	AssertFailed();
4098	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4099	}
4100	}
4101
4102	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4103	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4104	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4105	u32GuestCR4 \|= uSetCR4;
4106	u32GuestCR4 &= uZapCR4;
4107
4108	/* Write VT-x's view of the guest CR4 into the VMCS. */
4109	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4110	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4111	AssertRCReturn(rc, rc);
4112
4113	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4114	uint32_t u32CR4Mask = X86_CR4_VME
4115	\| X86_CR4_PAE
4116	\| X86_CR4_PGE
4117	\| X86_CR4_PSE
4118	\| X86_CR4_VMXE;
4119	if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4120	u32CR4Mask \|= X86_CR4_OSXSAVE;
4121	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4122	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4123	AssertRCReturn(rc, rc);
4124
4125	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4126	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4127
4128	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4129	}
4130	return rc;
4131	}
4132
4133
4134	/**
4135	* Loads the guest debug registers into the guest-state area in the VMCS.
4136	*
4137	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4138	*
4139	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4140	*
4141	* @returns VBox status code.
4142	* @param pVCpu The cross context virtual CPU structure.
4143	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4144	* out-of-sync. Make sure to update the required fields
4145	* before using them.
4146	*
4147	* @remarks No-long-jump zone!!!
4148	*/
4149	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4150	{
4151	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4152	return VINF_SUCCESS;
4153
4154	#ifdef VBOX_STRICT
4155	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4156	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4157	{
4158	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4159	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4160	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4161	}
4162	#endif
4163
4164	int rc;
4165	PVM pVM = pVCpu->CTX_SUFF(pVM);
4166	bool fSteppingDB = false;
4167	bool fInterceptMovDRx = false;
4168	if (pVCpu->hm.s.fSingleInstruction)
4169	{
4170	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4171	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4172	{
4173	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4174	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4175	AssertRCReturn(rc, rc);
4176	Assert(fSteppingDB == false);
4177	}
4178	else
4179	{
4180	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
4181	pVCpu->hm.s.fClearTrapFlag = true;
4182	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4183	fSteppingDB = true;
4184	}
4185	}
4186
4187	if ( fSteppingDB
4188	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4189	{
4190	/*
4191	* Use the combined guest and host DRx values found in the hypervisor
4192	* register set because the debugger has breakpoints active or someone
4193	* is single stepping on the host side without a monitor trap flag.
4194	*
4195	* Note! DBGF expects a clean DR6 state before executing guest code.
4196	*/
4197	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4198	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4199	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
4200	{
4201	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4202	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4203	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4204	}
4205	else
4206	#endif
4207	if (!CPUMIsHyperDebugStateActive(pVCpu))
4208	{
4209	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4210	Assert(CPUMIsHyperDebugStateActive(pVCpu));
4211	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4212	}
4213
4214	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4215	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4216	AssertRCReturn(rc, rc);
4217
4218	pVCpu->hm.s.fUsingHyperDR7 = true;
4219	fInterceptMovDRx = true;
4220	}
4221	else
4222	{
4223	/*
4224	* If the guest has enabled debug registers, we need to load them prior to
4225	* executing guest code so they'll trigger at the right time.
4226	*/
4227	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
4228	{
4229	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4230	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4231	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
4232	{
4233	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4234	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4235	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4236	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4237	}
4238	else
4239	#endif
4240	if (!CPUMIsGuestDebugStateActive(pVCpu))
4241	{
4242	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4243	Assert(CPUMIsGuestDebugStateActive(pVCpu));
4244	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4245	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4246	}
4247	Assert(!fInterceptMovDRx);
4248	}
4249	/*
4250	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4251	* must intercept #DB in order to maintain a correct DR6 guest value, and
4252	* because we need to intercept it to prevent nested #DBs from hanging the
4253	* CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4254	*/
4255	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4256	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4257	&& !CPUMIsGuestDebugStateActive(pVCpu))
4258	#else
4259	else if (!CPUMIsGuestDebugStateActive(pVCpu))
4260	#endif
4261	{
4262	fInterceptMovDRx = true;
4263	}
4264
4265	/* Update guest DR7. */
4266	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4267	AssertRCReturn(rc, rc);
4268
4269	pVCpu->hm.s.fUsingHyperDR7 = false;
4270	}
4271
4272	/*
4273	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4274	*/
4275	if (fInterceptMovDRx)
4276	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4277	else
4278	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4279	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4280	AssertRCReturn(rc, rc);
4281
4282	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4283	return VINF_SUCCESS;
4284	}
4285
4286
4287	#ifdef VBOX_STRICT
4288	/**
4289	* Strict function to validate segment registers.
4290	*
4291	* @remarks ASSUMES CR0 is up to date.
4292	*/
4293	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4294	{
4295	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4296	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4297	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4298	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4299	&& ( !CPUMIsGuestInRealModeEx(pCtx)
4300	&& !CPUMIsGuestInV86ModeEx(pCtx)))
4301	{
4302	/* Protected mode checks */
4303	/* CS */
4304	Assert(pCtx->cs.Attr.n.u1Present);
4305	Assert(!(pCtx->cs.Attr.u & 0xf00));
4306	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4307	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4308	\|\| !(pCtx->cs.Attr.n.u1Granularity));
4309	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4310	\|\| (pCtx->cs.Attr.n.u1Granularity));
4311	/* CS cannot be loaded with NULL in protected mode. */
4312	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4313	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
4314	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4315	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
4316	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4317	else
4318	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4319	/* SS */
4320	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4321	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4322	if ( !(pCtx->cr0 & X86_CR0_PE)
4323	\|\| pCtx->cs.Attr.n.u4Type == 3)
4324	{
4325	Assert(!pCtx->ss.Attr.n.u2Dpl);
4326	}
4327	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4328	{
4329	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4330	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
4331	Assert(pCtx->ss.Attr.n.u1Present);
4332	Assert(!(pCtx->ss.Attr.u & 0xf00));
4333	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4334	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4335	\|\| !(pCtx->ss.Attr.n.u1Granularity));
4336	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4337	\|\| (pCtx->ss.Attr.n.u1Granularity));
4338	}
4339	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4340	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4341	{
4342	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4343	Assert(pCtx->ds.Attr.n.u1Present);
4344	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4345	Assert(!(pCtx->ds.Attr.u & 0xf00));
4346	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4347	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4348	\|\| !(pCtx->ds.Attr.n.u1Granularity));
4349	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4350	\|\| (pCtx->ds.Attr.n.u1Granularity));
4351	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4352	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4353	}
4354	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4355	{
4356	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4357	Assert(pCtx->es.Attr.n.u1Present);
4358	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4359	Assert(!(pCtx->es.Attr.u & 0xf00));
4360	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4361	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4362	\|\| !(pCtx->es.Attr.n.u1Granularity));
4363	Assert( !(pCtx->es.u32Limit & 0xfff00000)
4364	\|\| (pCtx->es.Attr.n.u1Granularity));
4365	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4366	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4367	}
4368	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4369	{
4370	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4371	Assert(pCtx->fs.Attr.n.u1Present);
4372	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4373	Assert(!(pCtx->fs.Attr.u & 0xf00));
4374	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4375	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4376	\|\| !(pCtx->fs.Attr.n.u1Granularity));
4377	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4378	\|\| (pCtx->fs.Attr.n.u1Granularity));
4379	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4380	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4381	}
4382	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4383	{
4384	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4385	Assert(pCtx->gs.Attr.n.u1Present);
4386	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4387	Assert(!(pCtx->gs.Attr.u & 0xf00));
4388	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4389	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4390	\|\| !(pCtx->gs.Attr.n.u1Granularity));
4391	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4392	\|\| (pCtx->gs.Attr.n.u1Granularity));
4393	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4394	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4395	}
4396	/* 64-bit capable CPUs. */
4397	# if HC_ARCH_BITS == 64
4398	Assert(!(pCtx->cs.u64Base >> 32));
4399	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
4400	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
4401	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
4402	# endif
4403	}
4404	else if ( CPUMIsGuestInV86ModeEx(pCtx)
4405	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
4406	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
4407	{
4408	/* Real and v86 mode checks. */
4409	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4410	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4411	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4412	{
4413	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4414	}
4415	else
4416	{
4417	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4418	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4419	}
4420
4421	/* CS */
4422	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4423	Assert(pCtx->cs.u32Limit == 0xffff);
4424	Assert(u32CSAttr == 0xf3);
4425	/* SS */
4426	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4427	Assert(pCtx->ss.u32Limit == 0xffff);
4428	Assert(u32SSAttr == 0xf3);
4429	/* DS */
4430	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4431	Assert(pCtx->ds.u32Limit == 0xffff);
4432	Assert(u32DSAttr == 0xf3);
4433	/* ES */
4434	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4435	Assert(pCtx->es.u32Limit == 0xffff);
4436	Assert(u32ESAttr == 0xf3);
4437	/* FS */
4438	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4439	Assert(pCtx->fs.u32Limit == 0xffff);
4440	Assert(u32FSAttr == 0xf3);
4441	/* GS */
4442	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4443	Assert(pCtx->gs.u32Limit == 0xffff);
4444	Assert(u32GSAttr == 0xf3);
4445	/* 64-bit capable CPUs. */
4446	# if HC_ARCH_BITS == 64
4447	Assert(!(pCtx->cs.u64Base >> 32));
4448	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
4449	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
4450	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
4451	# endif
4452	}
4453	}
4454	#endif /* VBOX_STRICT */
4455
4456
4457	/**
4458	* Writes a guest segment register into the guest-state area in the VMCS.
4459	*
4460	* @returns VBox status code.
4461	* @param pVCpu The cross context virtual CPU structure.
4462	* @param idxSel Index of the selector in the VMCS.
4463	* @param idxLimit Index of the segment limit in the VMCS.
4464	* @param idxBase Index of the segment base in the VMCS.
4465	* @param idxAccess Index of the access rights of the segment in the VMCS.
4466	* @param pSelReg Pointer to the segment selector.
4467	*
4468	* @remarks No-long-jump zone!!!
4469	*/
4470	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4471	uint32_t idxAccess, PCPUMSELREG pSelReg)
4472	{
4473	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4474	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4475	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4476	AssertRCReturn(rc, rc);
4477
4478	uint32_t u32Access = pSelReg->Attr.u;
4479	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4480	{
4481	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4482	u32Access = 0xf3;
4483	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4484	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4485	}
4486	else
4487	{
4488	/*
4489	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4490	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4491	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4492	* loaded in protected-mode have their attribute as 0.
4493	*/
4494	if (!u32Access)
4495	u32Access = X86DESCATTR_UNUSABLE;
4496	}
4497
4498	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4499	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
4500	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4501
4502	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4503	AssertRCReturn(rc, rc);
4504	return rc;
4505	}
4506
4507
4508	/**
4509	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4510	* into the guest-state area in the VMCS.
4511	*
4512	* @returns VBox status code.
4513	* @param pVCpu The cross context virtual CPU structure.
4514	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4515	* out-of-sync. Make sure to update the required fields
4516	* before using them.
4517	*
4518	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4519	* @remarks No-long-jump zone!!!
4520	*/
4521	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4522	{
4523	int rc = VERR_INTERNAL_ERROR_5;
4524	PVM pVM = pVCpu->CTX_SUFF(pVM);
4525
4526	/*
4527	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4528	*/
4529	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4530	{
4531	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4532	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4533	{
4534	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4535	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4536	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4537	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4538	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4539	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4540	}
4541
4542	#ifdef VBOX_WITH_REM
4543	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4544	{
4545	Assert(pVM->hm.s.vmx.pRealModeTSS);
4546	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4547	if ( pVCpu->hm.s.vmx.fWasInRealMode
4548	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4549	{
4550	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4551	in real-mode (e.g. OpenBSD 4.0) */
4552	REMFlushTBs(pVM);
4553	Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4554	pVCpu->hm.s.vmx.fWasInRealMode = false;
4555	}
4556	}
4557	#endif
4558	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4559	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4560	AssertRCReturn(rc, rc);
4561	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4562	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4563	AssertRCReturn(rc, rc);
4564	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4565	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4566	AssertRCReturn(rc, rc);
4567	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4568	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4569	AssertRCReturn(rc, rc);
4570	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4571	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4572	AssertRCReturn(rc, rc);
4573	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4574	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4575	AssertRCReturn(rc, rc);
4576
4577	#ifdef VBOX_STRICT
4578	/* Validate. */
4579	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4580	#endif
4581
4582	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4583	Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4584	pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4585	}
4586
4587	/*
4588	* Guest TR.
4589	*/
4590	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4591	{
4592	/*
4593	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4594	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4595	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4596	*/
4597	uint16_t u16Sel = 0;
4598	uint32_t u32Limit = 0;
4599	uint64_t u64Base = 0;
4600	uint32_t u32AccessRights = 0;
4601
4602	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4603	{
4604	u16Sel = pMixedCtx->tr.Sel;
4605	u32Limit = pMixedCtx->tr.u32Limit;
4606	u64Base = pMixedCtx->tr.u64Base;
4607	u32AccessRights = pMixedCtx->tr.Attr.u;
4608	}
4609	else
4610	{
4611	Assert(pVM->hm.s.vmx.pRealModeTSS);
4612	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4613
4614	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4615	RTGCPHYS GCPhys;
4616	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4617	AssertRCReturn(rc, rc);
4618
4619	X86DESCATTR DescAttr;
4620	DescAttr.u = 0;
4621	DescAttr.n.u1Present = 1;
4622	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4623
4624	u16Sel = 0;
4625	u32Limit = HM_VTX_TSS_SIZE;
4626	u64Base = GCPhys; /* in real-mode phys = virt. */
4627	u32AccessRights = DescAttr.u;
4628	}
4629
4630	/* Validate. */
4631	Assert(!(u16Sel & RT_BIT(2)));
4632	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4633	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4634	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4635	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4636	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4637	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4638	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4639	Assert( (u32Limit & 0xfff) == 0xfff
4640	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4641	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4642	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4643
4644	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4645	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4646	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4647	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4648	AssertRCReturn(rc, rc);
4649
4650	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4651	Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4652	}
4653
4654	/*
4655	* Guest GDTR.
4656	*/
4657	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4658	{
4659	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4660	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4661	AssertRCReturn(rc, rc);
4662
4663	/* Validate. */
4664	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4665
4666	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4667	Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4668	}
4669
4670	/*
4671	* Guest LDTR.
4672	*/
4673	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4674	{
4675	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4676	uint32_t u32Access = 0;
4677	if (!pMixedCtx->ldtr.Attr.u)
4678	u32Access = X86DESCATTR_UNUSABLE;
4679	else
4680	u32Access = pMixedCtx->ldtr.Attr.u;
4681
4682	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4683	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4684	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4685	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4686	AssertRCReturn(rc, rc);
4687
4688	/* Validate. */
4689	if (!(u32Access & X86DESCATTR_UNUSABLE))
4690	{
4691	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4692	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4693	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4694	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4695	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4696	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4697	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4698	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4699	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4700	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4701	}
4702
4703	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4704	Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4705	}
4706
4707	/*
4708	* Guest IDTR.
4709	*/
4710	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4711	{
4712	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4713	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4714	AssertRCReturn(rc, rc);
4715
4716	/* Validate. */
4717	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4718
4719	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4720	Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4721	}
4722
4723	return VINF_SUCCESS;
4724	}
4725
4726
4727	/**
4728	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4729	* areas.
4730	*
4731	* These MSRs will automatically be loaded to the host CPU on every successful
4732	* VM-entry and stored from the host CPU on every successful VM-exit. This also
4733	* creates/updates MSR slots for the host MSRs. The actual host MSR values are
4734	* -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4735	*
4736	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
4737	*
4738	* @returns VBox status code.
4739	* @param pVCpu The cross context virtual CPU structure.
4740	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4741	* out-of-sync. Make sure to update the required fields
4742	* before using them.
4743	*
4744	* @remarks No-long-jump zone!!!
4745	*/
4746	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4747	{
4748	AssertPtr(pVCpu);
4749	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4750
4751	/*
4752	* MSRs that we use the auto-load/store MSR area in the VMCS.
4753	*/
4754	PVM pVM = pVCpu->CTX_SUFF(pVM);
4755	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4756	{
4757	/* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4758	#if HC_ARCH_BITS == 32
4759	if (pVM->hm.s.fAllow64BitGuests)
4760	{
4761	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4762	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4763	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4764	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4765	AssertRCReturn(rc, rc);
4766	# ifdef LOG_ENABLED
4767	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4768	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4769	{
4770	Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4771	pMsr->u64Value));
4772	}
4773	# endif
4774	}
4775	#endif
4776	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4777	}
4778
4779	/*
4780	* Guest Sysenter MSRs.
4781	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4782	* VM-exits on WRMSRs for these MSRs.
4783	*/
4784	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4785	{
4786	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4787	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4788	}
4789
4790	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4791	{
4792	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4793	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4794	}
4795
4796	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4797	{
4798	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4799	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4800	}
4801
4802	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4803	{
4804	if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4805	{
4806	/*
4807	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4808	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4809	*/
4810	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4811	{
4812	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4813	AssertRCReturn(rc,rc);
4814	Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4815	}
4816	else
4817	{
4818	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4819	NULL /* pfAddedAndUpdated */);
4820	AssertRCReturn(rc, rc);
4821
4822	/* We need to intercept reads too, see @bugref{7386#c16}. */
4823	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4824	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4825	Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4826	pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4827	}
4828	}
4829	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4830	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4831	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4832	}
4833
4834	return VINF_SUCCESS;
4835	}
4836
4837
4838	/**
4839	* Loads the guest activity state into the guest-state area in the VMCS.
4840	*
4841	* @returns VBox status code.
4842	* @param pVCpu The cross context virtual CPU structure.
4843	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4844	* out-of-sync. Make sure to update the required fields
4845	* before using them.
4846	*
4847	* @remarks No-long-jump zone!!!
4848	*/
4849	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4850	{
4851	NOREF(pMixedCtx);
4852	/** @todo See if we can make use of other states, e.g.
4853	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4854	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4855	{
4856	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4857	AssertRCReturn(rc, rc);
4858
4859	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4860	}
4861	return VINF_SUCCESS;
4862	}
4863
4864
4865	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4866	/**
4867	* Check if guest state allows safe use of 32-bit switcher again.
4868	*
4869	* Segment bases and protected mode structures must be 32-bit addressable
4870	* because the 32-bit switcher will ignore high dword when writing these VMCS
4871	* fields. See @bugref{8432} for details.
4872	*
4873	* @returns true if safe, false if must continue to use the 64-bit switcher.
4874	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4875	* out-of-sync. Make sure to update the required fields
4876	* before using them.
4877	*
4878	* @remarks No-long-jump zone!!!
4879	*/
4880	static bool hmR0VmxIs32BitSwitcherSafe(PCPUMCTX pMixedCtx)
4881	{
4882	if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000))
4883	return false;
4884	if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000))
4885	return false;
4886	if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000))
4887	return false;
4888	if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000))
4889	return false;
4890	if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000))
4891	return false;
4892	if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000))
4893	return false;
4894	if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000))
4895	return false;
4896	if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000))
4897	return false;
4898	if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000))
4899	return false;
4900	if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000))
4901	return false;
4902	/* All good, bases are 32-bit. */
4903	return true;
4904	}
4905	#endif
4906
4907
4908	/**
4909	* Sets up the appropriate function to run guest code.
4910	*
4911	* @returns VBox status code.
4912	* @param pVCpu The cross context virtual CPU structure.
4913	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4914	* out-of-sync. Make sure to update the required fields
4915	* before using them.
4916	*
4917	* @remarks No-long-jump zone!!!
4918	*/
4919	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4920	{
4921	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4922	{
4923	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4924	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4925	#endif
4926	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4927	#if HC_ARCH_BITS == 32
4928	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4929	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4930	{
4931	if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4932	{
4933	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4934	AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4935	\| HM_CHANGED_VMX_ENTRY_CTLS
4936	\| HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4937	}
4938	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4939
4940	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4941	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4942	pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4943	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 64-bit switcher\n", pVCpu->idCpu));
4944	}
4945	#else
4946	/* 64-bit host. */
4947	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4948	#endif
4949	}
4950	else
4951	{
4952	/* Guest is not in long mode, use the 32-bit handler. */
4953	#if HC_ARCH_BITS == 32
4954	if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4955	&& !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4956	&& pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4957	{
4958	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4959	AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4960	\| HM_CHANGED_VMX_ENTRY_CTLS
4961	\| HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4962	}
4963	# ifdef VBOX_ENABLE_64_BITS_GUESTS
4964	/*
4965	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel design, see @bugref{8432#c7}.
4966	* If real-on-v86 mode is active, clear the 64-bit switcher flag because now we know the guest is in a sane
4967	* state where it's safe to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4968	* the much faster 32-bit switcher again.
4969	*/
4970	if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4971	{
4972	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4973	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher\n", pVCpu->idCpu));
4974	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4975	}
4976	else
4977	{
4978	Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4979	if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4980	\|\| hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
4981	{
4982	pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4983	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4984	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR
4985	\| HM_CHANGED_VMX_ENTRY_CTLS
4986	\| HM_CHANGED_VMX_EXIT_CTLS
4987	\| HM_CHANGED_HOST_CONTEXT);
4988	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher (safe)\n", pVCpu->idCpu));
4989	}
4990	}
4991	# else
4992	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4993	# endif
4994	#else
4995	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4996	#endif
4997	}
4998	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4999	return VINF_SUCCESS;
5000	}
5001
5002
5003	/**
5004	* Wrapper for running the guest code in VT-x.
5005	*
5006	* @returns VBox status code, no informational status codes.
5007	* @param pVM The cross context VM structure.
5008	* @param pVCpu The cross context virtual CPU structure.
5009	* @param pCtx Pointer to the guest-CPU context.
5010	*
5011	* @remarks No-long-jump zone!!!
5012	*/
5013	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
5014	{
5015	/*
5016	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
5017	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
5018	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
5019	*/
5020	bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
5021	/** @todo Add stats for resume vs launch. */
5022	#ifdef VBOX_WITH_KERNEL_USING_XMM
5023	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
5024	#else
5025	int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
5026	#endif
5027	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
5028	return rc;
5029	}
5030
5031
5032	/**
5033	* Reports world-switch error and dumps some useful debug info.
5034	*
5035	* @param pVM The cross context VM structure.
5036	* @param pVCpu The cross context virtual CPU structure.
5037	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5038	* @param pCtx Pointer to the guest-CPU context.
5039	* @param pVmxTransient Pointer to the VMX transient structure (only
5040	* exitReason updated).
5041	*/
5042	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
5043	{
5044	Assert(pVM);
5045	Assert(pVCpu);
5046	Assert(pCtx);
5047	Assert(pVmxTransient);
5048	HMVMX_ASSERT_PREEMPT_SAFE();
5049
5050	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
5051	switch (rcVMRun)
5052	{
5053	case VERR_VMX_INVALID_VMXON_PTR:
5054	AssertFailed();
5055	break;
5056	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5057	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5058	{
5059	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5060	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5061	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5062	AssertRC(rc);
5063
5064	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5065	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5066	Cannot do it here as we may have been long preempted. */
5067
5068	#ifdef VBOX_STRICT
5069	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5070	pVmxTransient->uExitReason));
5071	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5072	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5073	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5074	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5075	else
5076	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5077	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5078	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5079
5080	/* VMX control bits. */
5081	uint32_t u32Val;
5082	uint64_t u64Val;
5083	RTHCUINTREG uHCReg;
5084	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5085	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5086	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5087	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5088	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5089	{
5090	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5091	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5092	}
5093	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5094	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5095	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5096	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5097	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5098	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5099	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5100	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5101	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5102	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5103	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5104	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5105	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5106	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5107	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5108	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5109	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5110	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5111	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5112	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5113	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5114	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5115	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5116	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5117	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5118	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5119	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5120	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5121	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5122	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5123	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5124	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5125	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5126	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5127	if (pVM->hm.s.fNestedPaging)
5128	{
5129	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5130	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5131	}
5132
5133	/* Guest bits. */
5134	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5135	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5136	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5137	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5138	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5139	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5140	if (pVM->hm.s.vmx.fVpid)
5141	{
5142	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5143	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5144	}
5145
5146	/* Host bits. */
5147	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5148	Log4(("Host CR0 %#RHr\n", uHCReg));
5149	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5150	Log4(("Host CR3 %#RHr\n", uHCReg));
5151	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5152	Log4(("Host CR4 %#RHr\n", uHCReg));
5153
5154	RTGDTR HostGdtr;
5155	PCX86DESCHC pDesc;
5156	ASMGetGDTR(&HostGdtr);
5157	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5158	Log4(("Host CS %#08x\n", u32Val));
5159	if (u32Val < HostGdtr.cbGdt)
5160	{
5161	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5162	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5163	}
5164
5165	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5166	Log4(("Host DS %#08x\n", u32Val));
5167	if (u32Val < HostGdtr.cbGdt)
5168	{
5169	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5170	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5171	}
5172
5173	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5174	Log4(("Host ES %#08x\n", u32Val));
5175	if (u32Val < HostGdtr.cbGdt)
5176	{
5177	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5178	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5179	}
5180
5181	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5182	Log4(("Host FS %#08x\n", u32Val));
5183	if (u32Val < HostGdtr.cbGdt)
5184	{
5185	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5186	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5187	}
5188
5189	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5190	Log4(("Host GS %#08x\n", u32Val));
5191	if (u32Val < HostGdtr.cbGdt)
5192	{
5193	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5194	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5195	}
5196
5197	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5198	Log4(("Host SS %#08x\n", u32Val));
5199	if (u32Val < HostGdtr.cbGdt)
5200	{
5201	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5202	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5203	}
5204
5205	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5206	Log4(("Host TR %#08x\n", u32Val));
5207	if (u32Val < HostGdtr.cbGdt)
5208	{
5209	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5210	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5211	}
5212
5213	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5214	Log4(("Host TR Base %#RHv\n", uHCReg));
5215	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5216	Log4(("Host GDTR Base %#RHv\n", uHCReg));
5217	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5218	Log4(("Host IDTR Base %#RHv\n", uHCReg));
5219	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5220	Log4(("Host SYSENTER CS %#08x\n", u32Val));
5221	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5222	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5223	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5224	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5225	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5226	Log4(("Host RSP %#RHv\n", uHCReg));
5227	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5228	Log4(("Host RIP %#RHv\n", uHCReg));
5229	# if HC_ARCH_BITS == 64
5230	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5231	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5232	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5233	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5234	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5235	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5236	# endif
5237	#endif /* VBOX_STRICT */
5238	break;
5239	}
5240
5241	default:
5242	/* Impossible */
5243	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5244	break;
5245	}
5246	NOREF(pVM); NOREF(pCtx);
5247	}
5248
5249
5250	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5251	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5252	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5253	#endif
5254	#ifdef VBOX_STRICT
5255	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5256	{
5257	switch (idxField)
5258	{
5259	case VMX_VMCS_GUEST_RIP:
5260	case VMX_VMCS_GUEST_RSP:
5261	case VMX_VMCS_GUEST_SYSENTER_EIP:
5262	case VMX_VMCS_GUEST_SYSENTER_ESP:
5263	case VMX_VMCS_GUEST_GDTR_BASE:
5264	case VMX_VMCS_GUEST_IDTR_BASE:
5265	case VMX_VMCS_GUEST_CS_BASE:
5266	case VMX_VMCS_GUEST_DS_BASE:
5267	case VMX_VMCS_GUEST_ES_BASE:
5268	case VMX_VMCS_GUEST_FS_BASE:
5269	case VMX_VMCS_GUEST_GS_BASE:
5270	case VMX_VMCS_GUEST_SS_BASE:
5271	case VMX_VMCS_GUEST_LDTR_BASE:
5272	case VMX_VMCS_GUEST_TR_BASE:
5273	case VMX_VMCS_GUEST_CR3:
5274	return true;
5275	}
5276	return false;
5277	}
5278
5279	static bool hmR0VmxIsValidReadField(uint32_t idxField)
5280	{
5281	switch (idxField)
5282	{
5283	/* Read-only fields. */
5284	case VMX_VMCS_RO_EXIT_QUALIFICATION:
5285	return true;
5286	}
5287	/* Remaining readable fields should also be writable. */
5288	return hmR0VmxIsValidWriteField(idxField);
5289	}
5290	#endif /* VBOX_STRICT */
5291
5292
5293	/**
5294	* Executes the specified handler in 64-bit mode.
5295	*
5296	* @returns VBox status code (no informational status codes).
5297	* @param pVM The cross context VM structure.
5298	* @param pVCpu The cross context virtual CPU structure.
5299	* @param pCtx Pointer to the guest CPU context.
5300	* @param enmOp The operation to perform.
5301	* @param cParams Number of parameters.
5302	* @param paParam Array of 32-bit parameters.
5303	*/
5304	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5305	uint32_t cParams, uint32_t *paParam)
5306	{
5307	NOREF(pCtx);
5308
5309	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5310	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5311	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5312	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5313
5314	#ifdef VBOX_STRICT
5315	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5316	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5317
5318	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5319	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5320	#endif
5321
5322	/* Disable interrupts. */
5323	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5324
5325	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5326	RTCPUID idHostCpu = RTMpCpuId();
5327	CPUMR0SetLApic(pVCpu, idHostCpu);
5328	#endif
5329
5330	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5331	RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5332
5333	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5334	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5335	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5336
5337	/* Leave VMX Root Mode. */
5338	VMXDisable();
5339
5340	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5341
5342	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5343	CPUMSetHyperEIP(pVCpu, enmOp);
5344	for (int i = (int)cParams - 1; i >= 0; i--)
5345	CPUMPushHyper(pVCpu, paParam[i]);
5346
5347	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5348
5349	/* Call the switcher. */
5350	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5351	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5352
5353	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5354	/* Make sure the VMX instructions don't cause #UD faults. */
5355	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5356
5357	/* Re-enter VMX Root Mode */
5358	int rc2 = VMXEnable(HCPhysCpuPage);
5359	if (RT_FAILURE(rc2))
5360	{
5361	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5362	ASMSetFlags(fOldEFlags);
5363	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5364	return rc2;
5365	}
5366
5367	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5368	AssertRC(rc2);
5369	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5370	Assert(!(ASMGetFlags() & X86_EFL_IF));
5371	ASMSetFlags(fOldEFlags);
5372	return rc;
5373	}
5374
5375
5376	/**
5377	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5378	* supporting 64-bit guests.
5379	*
5380	* @returns VBox status code.
5381	* @param fResume Whether to VMLAUNCH or VMRESUME.
5382	* @param pCtx Pointer to the guest-CPU context.
5383	* @param pCache Pointer to the VMCS cache.
5384	* @param pVM The cross context VM structure.
5385	* @param pVCpu The cross context virtual CPU structure.
5386	*/
5387	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5388	{
5389	NOREF(fResume);
5390
5391	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5392	RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5393
5394	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5395	pCache->uPos = 1;
5396	pCache->interPD = PGMGetInterPaeCR3(pVM);
5397	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5398	#endif
5399
5400	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5401	pCache->TestIn.HCPhysCpuPage = 0;
5402	pCache->TestIn.HCPhysVmcs = 0;
5403	pCache->TestIn.pCache = 0;
5404	pCache->TestOut.HCPhysVmcs = 0;
5405	pCache->TestOut.pCache = 0;
5406	pCache->TestOut.pCtx = 0;
5407	pCache->TestOut.eflags = 0;
5408	#else
5409	NOREF(pCache);
5410	#endif
5411
5412	uint32_t aParam[10];
5413	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5414	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5415	aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5416	aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5417	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5418	aParam[5] = 0;
5419	aParam[6] = VM_RC_ADDR(pVM, pVM);
5420	aParam[7] = 0;
5421	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5422	aParam[9] = 0;
5423
5424	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5425	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5426	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5427	#endif
5428	int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5429
5430	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5431	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5432	Assert(pCtx->dr[4] == 10);
5433	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5434	#endif
5435
5436	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5437	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5438	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5439	pVCpu->hm.s.vmx.HCPhysVmcs));
5440	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5441	pCache->TestOut.HCPhysVmcs));
5442	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5443	pCache->TestOut.pCache));
5444	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5445	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5446	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5447	pCache->TestOut.pCtx));
5448	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5449	#endif
5450	return rc;
5451	}
5452
5453
5454	/**
5455	* Initialize the VMCS-Read cache.
5456	*
5457	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5458	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5459	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5460	* (those that have a 32-bit FULL & HIGH part).
5461	*
5462	* @returns VBox status code.
5463	* @param pVM The cross context VM structure.
5464	* @param pVCpu The cross context virtual CPU structure.
5465	*/
5466	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5467	{
5468	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5469	{ \
5470	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5471	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5472	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5473	++cReadFields; \
5474	}
5475
5476	AssertPtr(pVM);
5477	AssertPtr(pVCpu);
5478	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5479	uint32_t cReadFields = 0;
5480
5481	/*
5482	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
5483	* and serve to indicate exceptions to the rules.
5484	*/
5485
5486	/* Guest-natural selector base fields. */
5487	#if 0
5488	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5489	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5490	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5491	#endif
5492	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5493	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5494	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5495	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5496	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5497	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5498	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5499	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5500	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5501	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5502	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5503	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5504	#if 0
5505	/* Unused natural width guest-state fields. */
5506	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5507	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5508	#endif
5509	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5510	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5511
5512	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5513	#if 0
5514	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5515	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5516	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5517	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5518	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5519	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5520	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5521	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5522	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5523	#endif
5524
5525	/* Natural width guest-state fields. */
5526	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5527	#if 0
5528	/* Currently unused field. */
5529	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5530	#endif
5531
5532	if (pVM->hm.s.fNestedPaging)
5533	{
5534	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5535	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5536	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5537	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5538	}
5539	else
5540	{
5541	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5542	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5543	}
5544
5545	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5546	return VINF_SUCCESS;
5547	}
5548
5549
5550	/**
5551	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5552	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5553	* darwin, running 64-bit guests).
5554	*
5555	* @returns VBox status code.
5556	* @param pVCpu The cross context virtual CPU structure.
5557	* @param idxField The VMCS field encoding.
5558	* @param u64Val 16, 32 or 64-bit value.
5559	*/
5560	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5561	{
5562	int rc;
5563	switch (idxField)
5564	{
5565	/*
5566	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5567	*/
5568	/* 64-bit Control fields. */
5569	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5570	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5571	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5572	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5573	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5574	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5575	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5576	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5577	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5578	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5579	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5580	case VMX_VMCS64_CTRL_EPTP_FULL:
5581	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5582	/* 64-bit Guest-state fields. */
5583	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5584	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5585	case VMX_VMCS64_GUEST_PAT_FULL:
5586	case VMX_VMCS64_GUEST_EFER_FULL:
5587	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5588	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5589	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5590	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5591	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5592	/* 64-bit Host-state fields. */
5593	case VMX_VMCS64_HOST_PAT_FULL:
5594	case VMX_VMCS64_HOST_EFER_FULL:
5595	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5596	{
5597	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5598	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5599	break;
5600	}
5601
5602	/*
5603	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5604	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5605	*/
5606	/* Natural-width Guest-state fields. */
5607	case VMX_VMCS_GUEST_CR3:
5608	case VMX_VMCS_GUEST_ES_BASE:
5609	case VMX_VMCS_GUEST_CS_BASE:
5610	case VMX_VMCS_GUEST_SS_BASE:
5611	case VMX_VMCS_GUEST_DS_BASE:
5612	case VMX_VMCS_GUEST_FS_BASE:
5613	case VMX_VMCS_GUEST_GS_BASE:
5614	case VMX_VMCS_GUEST_LDTR_BASE:
5615	case VMX_VMCS_GUEST_TR_BASE:
5616	case VMX_VMCS_GUEST_GDTR_BASE:
5617	case VMX_VMCS_GUEST_IDTR_BASE:
5618	case VMX_VMCS_GUEST_RSP:
5619	case VMX_VMCS_GUEST_RIP:
5620	case VMX_VMCS_GUEST_SYSENTER_ESP:
5621	case VMX_VMCS_GUEST_SYSENTER_EIP:
5622	{
5623	if (!(RT_HI_U32(u64Val)))
5624	{
5625	/* If this field is 64-bit, VT-x will zero out the top bits. */
5626	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5627	}
5628	else
5629	{
5630	/* Assert that only the 32->64 switcher case should ever come here. */
5631	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5632	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5633	}
5634	break;
5635	}
5636
5637	default:
5638	{
5639	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5640	rc = VERR_INVALID_PARAMETER;
5641	break;
5642	}
5643	}
5644	AssertRCReturn(rc, rc);
5645	return rc;
5646	}
5647
5648
5649	/**
5650	* Queue up a VMWRITE by using the VMCS write cache.
5651	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5652	*
5653	* @param pVCpu The cross context virtual CPU structure.
5654	* @param idxField The VMCS field encoding.
5655	* @param u64Val 16, 32 or 64-bit value.
5656	*/
5657	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5658	{
5659	AssertPtr(pVCpu);
5660	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5661
5662	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5663	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5664
5665	/* Make sure there are no duplicates. */
5666	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5667	{
5668	if (pCache->Write.aField[i] == idxField)
5669	{
5670	pCache->Write.aFieldVal[i] = u64Val;
5671	return VINF_SUCCESS;
5672	}
5673	}
5674
5675	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5676	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5677	pCache->Write.cValidEntries++;
5678	return VINF_SUCCESS;
5679	}
5680	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5681
5682
5683	/**
5684	* Sets up the usage of TSC-offsetting and updates the VMCS.
5685	*
5686	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5687	* VMX preemption timer.
5688	*
5689	* @returns VBox status code.
5690	* @param pVM The cross context VM structure.
5691	* @param pVCpu The cross context virtual CPU structure.
5692	*
5693	* @remarks No-long-jump zone!!!
5694	*/
5695	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5696	{
5697	int rc;
5698	bool fOffsettedTsc;
5699	bool fParavirtTsc;
5700	if (pVM->hm.s.vmx.fUsePreemptTimer)
5701	{
5702	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5703	&fOffsettedTsc, &fParavirtTsc);
5704
5705	/* Make sure the returned values have sane upper and lower boundaries. */
5706	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5707	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5708	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5709	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5710
5711	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5712	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5713	}
5714	else
5715	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5716
5717	/** @todo later optimize this to be done elsewhere and not before every
5718	* VM-entry. */
5719	if (fParavirtTsc)
5720	{
5721	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5722	information before every VM-entry, hence disable it for performance sake. */
5723	#if 0
5724	rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5725	AssertRC(rc);
5726	#endif
5727	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5728	}
5729
5730	if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5731	{
5732	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5733	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5734
5735	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5736	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5737	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5738	}
5739	else
5740	{
5741	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5742	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5743	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5744	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5745	}
5746	}
5747
5748
5749	#ifdef HMVMX_USE_IEM_EVENT_REFLECTION
5750	/**
5751	* Gets the IEM exception flags for the specified vector and IDT vectoring /
5752	* VM-exit interruption info type.
5753	*
5754	* @returns The IEM exception flags.
5755	* @param uVector The event vector.
5756	* @param uVmxVectorType The VMX event type.
5757	*
5758	* @remarks This function currently only constructs flags required for
5759	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5760	* and CR2 aspects of an exception are not included).
5761	*/
5762	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5763	{
5764	uint32_t fIemXcptFlags;
5765	switch (uVmxVectorType)
5766	{
5767	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5768	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5769	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5770	break;
5771
5772	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5773	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5774	break;
5775
5776	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5777	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
5778	break;
5779
5780	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5781	{
5782	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5783	if (uVector == X86_XCPT_BP)
5784	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
5785	else if (uVector == X86_XCPT_OF)
5786	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
5787	else
5788	{
5789	fIemXcptFlags = 0;
5790	AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5791	}
5792	break;
5793	}
5794
5795	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5796	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5797	break;
5798
5799	default:
5800	fIemXcptFlags = 0;
5801	AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5802	break;
5803	}
5804	return fIemXcptFlags;
5805	}
5806
5807	#else
5808	/**
5809	* Determines if an exception is a contributory exception.
5810	*
5811	* Contributory exceptions are ones which can cause double-faults unless the
5812	* original exception was a benign exception. Page-fault is intentionally not
5813	* included here as it's a conditional contributory exception.
5814	*
5815	* @returns true if the exception is contributory, false otherwise.
5816	* @param uVector The exception vector.
5817	*/
5818	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5819	{
5820	switch (uVector)
5821	{
5822	case X86_XCPT_GP:
5823	case X86_XCPT_SS:
5824	case X86_XCPT_NP:
5825	case X86_XCPT_TS:
5826	case X86_XCPT_DE:
5827	return true;
5828	default:
5829	break;
5830	}
5831	return false;
5832	}
5833	#endif /* HMVMX_USE_IEM_EVENT_REFLECTION */
5834
5835
5836	/**
5837	* Sets an event as a pending event to be injected into the guest.
5838	*
5839	* @param pVCpu The cross context virtual CPU structure.
5840	* @param u32IntInfo The VM-entry interruption-information field.
5841	* @param cbInstr The VM-entry instruction length in bytes (for software
5842	* interrupts, exceptions and privileged software
5843	* exceptions).
5844	* @param u32ErrCode The VM-entry exception error code.
5845	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5846	* page-fault.
5847	*
5848	* @remarks Statistics counter assumes this is a guest event being injected or
5849	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5850	* always incremented.
5851	*/
5852	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5853	RTGCUINTPTR GCPtrFaultAddress)
5854	{
5855	Assert(!pVCpu->hm.s.Event.fPending);
5856	pVCpu->hm.s.Event.fPending = true;
5857	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5858	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5859	pVCpu->hm.s.Event.cbInstr = cbInstr;
5860	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5861	}
5862
5863
5864	/**
5865	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5866	*
5867	* @param pVCpu The cross context virtual CPU structure.
5868	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5869	* out-of-sync. Make sure to update the required fields
5870	* before using them.
5871	*/
5872	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5873	{
5874	NOREF(pMixedCtx);
5875	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5876	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5877	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5878	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5879	}
5880
5881
5882	/**
5883	* Handle a condition that occurred while delivering an event through the guest
5884	* IDT.
5885	*
5886	* @returns Strict VBox status code (i.e. informational status codes too).
5887	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5888	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5889	* to continue execution of the guest which will delivery the \#DF.
5890	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5891	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5892	*
5893	* @param pVCpu The cross context virtual CPU structure.
5894	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5895	* out-of-sync. Make sure to update the required fields
5896	* before using them.
5897	* @param pVmxTransient Pointer to the VMX transient structure.
5898	*
5899	* @remarks No-long-jump zone!!!
5900	*/
5901	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5902	{
5903	uint32_t const uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5904
5905	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5906	rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5907
5908	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5909	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5910	{
5911	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5912	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5913	#ifdef HMVMX_USE_IEM_EVENT_REFLECTION
5914	/*
5915	* If the event was a software interrupt (generated with INT n) or a software exception (generated
5916	* by INT3/INTO) or a privileged software exception (generated by INT1), we can handle the VM-exit
5917	* and continue guest execution which will re-execute the instruction rather than re-injecting the
5918	* exception, as that can cause premature trips to ring-3 before injection and involve TRPM which
5919	* currently has no way of storing that these exceptions were caused by these instructions
5920	* (ICEBP's #DB poses the problem).
5921	*/
5922	IEMXCPTRAISE enmRaise;
5923	IEMXCPTRAISEINFO fRaiseInfo;
5924	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5925	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5926	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5927	{
5928	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5929	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5930	}
5931	else if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5932	{
5933	uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5934	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5935	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5936	/** @todo Make AssertMsgReturn as just AssertMsg later. */
5937	AssertMsgReturn(uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT,
5938	("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5939	uExitVectorType), VERR_VMX_IPE_5);
5940	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5941
5942	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5943	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
5944	{
5945	pVmxTransient->fVectoringPF = true;
5946	enmRaise = IEMXCPTRAISE_PREV_EVENT;
5947	}
5948	}
5949	else
5950	{
5951	/*
5952	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5953	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5954	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
5955	*/
5956	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5957	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5958	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5959	enmRaise = IEMXCPTRAISE_PREV_EVENT;
5960	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5961	}
5962
5963	/*
5964	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5965	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5966	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5967	* subsequent VM-entry would fail.
5968	*
5969	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5970	*/
5971	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
5972	&& uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5973	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
5974	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
5975	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5976	{
5977	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5978	}
5979
5980	switch (enmRaise)
5981	{
5982	case IEMXCPTRAISE_CURRENT_XCPT:
5983	{
5984	Log4(("IDT: vcpu[%RU32] Pending secondary xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n", pVCpu->idCpu,
5985	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
5986	Assert(rcStrict == VINF_SUCCESS);
5987	break;
5988	}
5989
5990	case IEMXCPTRAISE_PREV_EVENT:
5991	{
5992	uint32_t u32ErrCode;
5993	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5994	{
5995	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5996	AssertRCReturn(rc2, rc2);
5997	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5998	}
5999	else
6000	u32ErrCode = 0;
6001
6002	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
6003	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6004	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6005	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
6006
6007	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
6008	pVCpu->hm.s.Event.u32ErrCode));
6009	Assert(rcStrict == VINF_SUCCESS);
6010	break;
6011	}
6012
6013	case IEMXCPTRAISE_REEXEC_INSTR:
6014	Assert(rcStrict == VINF_SUCCESS);
6015	break;
6016
6017	case IEMXCPTRAISE_DOUBLE_FAULT:
6018	{
6019	/*
6020	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
6021	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
6022	*/
6023	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6024	{
6025	pVmxTransient->fVectoringDoublePF = true;
6026	Log4(("IDT: vcpu[%RU32] Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
6027	pMixedCtx->cr2));
6028	rcStrict = VINF_SUCCESS;
6029	}
6030	else
6031	{
6032	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6033	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
6034	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
6035	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
6036	rcStrict = VINF_HM_DOUBLE_FAULT;
6037	}
6038	break;
6039	}
6040
6041	case IEMXCPTRAISE_TRIPLE_FAULT:
6042	{
6043	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
6044	uExitVector));
6045	rcStrict = VINF_EM_RESET;
6046	break;
6047	}
6048
6049	case IEMXCPTRAISE_CPU_HANG:
6050	{
6051	Log4(("IDT: vcpu[%RU32] Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", pVCpu->idCpu, fRaiseInfo));
6052	rcStrict = VERR_EM_GUEST_CPU_HANG;
6053	break;
6054	}
6055
6056	default:
6057	{
6058	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
6059	rcStrict = VERR_VMX_IPE_2;
6060	break;
6061	}
6062	}
6063	#else
6064	typedef enum
6065	{
6066	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
6067	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
6068	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
6069	VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
6070	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
6071	} VMXREFLECTXCPT;
6072
6073	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
6074	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
6075	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
6076	{
6077	if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
6078	{
6079	enmReflect = VMXREFLECTXCPT_XCPT;
6080	#ifdef VBOX_STRICT
6081	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
6082	&& uExitVector == X86_XCPT_PF)
6083	{
6084	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6085	}
6086	#endif
6087	if ( uExitVector == X86_XCPT_PF
6088	&& uIdtVector == X86_XCPT_PF)
6089	{
6090	pVmxTransient->fVectoringDoublePF = true;
6091	Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6092	}
6093	else if ( uExitVector == X86_XCPT_AC
6094	&& uIdtVector == X86_XCPT_AC)
6095	{
6096	enmReflect = VMXREFLECTXCPT_HANG;
6097	Log4(("IDT: Nested #AC - Bad guest\n"));
6098	}
6099	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
6100	&& hmR0VmxIsContributoryXcpt(uExitVector)
6101	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
6102	\|\| uIdtVector == X86_XCPT_PF))
6103	{
6104	enmReflect = VMXREFLECTXCPT_DF;
6105	}
6106	else if (uIdtVector == X86_XCPT_DF)
6107	enmReflect = VMXREFLECTXCPT_TF;
6108	}
6109	else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
6110	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
6111	{
6112	/*
6113	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
6114	* privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
6115	*/
6116	enmReflect = VMXREFLECTXCPT_XCPT;
6117
6118	if (uExitVector == X86_XCPT_PF)
6119	{
6120	pVmxTransient->fVectoringPF = true;
6121	Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6122	}
6123	}
6124	}
6125	else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6126	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
6127	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
6128	{
6129	/*
6130	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
6131	* interruption-information will not be valid as it's not an exception and we end up here. In such cases,
6132	* it is sufficient to reflect the original exception to the guest after handling the VM-exit.
6133	*/
6134	enmReflect = VMXREFLECTXCPT_XCPT;
6135	}
6136
6137	/*
6138	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
6139	* while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
6140	* re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
6141	*
6142	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
6143	*/
6144	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6145	&& enmReflect == VMXREFLECTXCPT_XCPT
6146	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
6147	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6148	{
6149	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6150	}
6151
6152	switch (enmReflect)
6153	{
6154	case VMXREFLECTXCPT_XCPT:
6155	{
6156	Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6157	&& uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6158	&& uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
6159
6160	uint32_t u32ErrCode = 0;
6161	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6162	{
6163	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6164	AssertRCReturn(rc2, rc2);
6165	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6166	}
6167
6168	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
6169	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6170	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6171	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
6172	rcStrict = VINF_SUCCESS;
6173	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
6174	pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
6175
6176	break;
6177	}
6178
6179	case VMXREFLECTXCPT_DF:
6180	{
6181	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6182	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
6183	rcStrict = VINF_HM_DOUBLE_FAULT;
6184	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
6185	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
6186
6187	break;
6188	}
6189
6190	case VMXREFLECTXCPT_TF:
6191	{
6192	rcStrict = VINF_EM_RESET;
6193	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
6194	uExitVector));
6195	break;
6196	}
6197
6198	case VMXREFLECTXCPT_HANG:
6199	{
6200	rcStrict = VERR_EM_GUEST_CPU_HANG;
6201	break;
6202	}
6203
6204	default:
6205	Assert(rcStrict == VINF_SUCCESS);
6206	break;
6207	}
6208	#endif /* HMVMX_USE_IEM_EVENT_REFLECTION */
6209	}
6210	else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6211	&& VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6212	&& uExitVector != X86_XCPT_DF
6213	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6214	{
6215	/*
6216	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6217	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6218	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6219	*/
6220	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6221	{
6222	Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
6223	pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6224	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6225	}
6226	}
6227
6228	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
6229	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
6230	return rcStrict;
6231	}
6232
6233
6234	/**
6235	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
6236	*
6237	* @returns VBox status code.
6238	* @param pVCpu The cross context virtual CPU structure.
6239	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6240	* out-of-sync. Make sure to update the required fields
6241	* before using them.
6242	*
6243	* @remarks No-long-jump zone!!!
6244	*/
6245	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6246	{
6247	NOREF(pMixedCtx);
6248
6249	/*
6250	* While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
6251	* see hmR0VmxLeave(). Safer to just make this code non-preemptible.
6252	*/
6253	VMMRZCallRing3Disable(pVCpu);
6254	HM_DISABLE_PREEMPT();
6255
6256	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
6257	{
6258	#ifndef DEBUG_bird /** @todo this triggers running bs3-cpu-generated-1.img with --debug-command-line
6259	* and 'dbgc-init' containing:
6260	* sxe "xcpt_de"
6261	* sxe "xcpt_bp"
6262	* sxi "xcpt_gp"
6263	* sxi "xcpt_ss"
6264	* sxi "xcpt_np"
6265	*/
6266	/** @todo r=ramshankar: Should be fixed after r119291. */
6267	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
6268	#endif
6269	uint32_t uVal = 0;
6270	uint32_t uShadow = 0;
6271	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
6272	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
6273	AssertRCReturn(rc, rc);
6274
6275	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
6276	CPUMSetGuestCR0(pVCpu, uVal);
6277	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
6278	}
6279
6280	HM_RESTORE_PREEMPT();
6281	VMMRZCallRing3Enable(pVCpu);
6282	return VINF_SUCCESS;
6283	}
6284
6285
6286	/**
6287	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
6288	*
6289	* @returns VBox status code.
6290	* @param pVCpu The cross context virtual CPU structure.
6291	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6292	* out-of-sync. Make sure to update the required fields
6293	* before using them.
6294	*
6295	* @remarks No-long-jump zone!!!
6296	*/
6297	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6298	{
6299	NOREF(pMixedCtx);
6300
6301	int rc = VINF_SUCCESS;
6302	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
6303	{
6304	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4));
6305	uint32_t uVal = 0;
6306	uint32_t uShadow = 0;
6307	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
6308	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
6309	AssertRCReturn(rc, rc);
6310
6311	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
6312	CPUMSetGuestCR4(pVCpu, uVal);
6313	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6314	}
6315	return rc;
6316	}
6317
6318
6319	/**
6320	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
6321	*
6322	* @returns VBox status code.
6323	* @param pVCpu The cross context virtual CPU structure.
6324	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6325	* out-of-sync. Make sure to update the required fields
6326	* before using them.
6327	*
6328	* @remarks No-long-jump zone!!!
6329	*/
6330	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6331	{
6332	int rc = VINF_SUCCESS;
6333	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6334	{
6335	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP));
6336	uint64_t u64Val = 0;
6337	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6338	AssertRCReturn(rc, rc);
6339
6340	pMixedCtx->rip = u64Val;
6341	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6342	}
6343	return rc;
6344	}
6345
6346
6347	/**
6348	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
6349	*
6350	* @returns VBox status code.
6351	* @param pVCpu The cross context virtual CPU structure.
6352	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6353	* out-of-sync. Make sure to update the required fields
6354	* before using them.
6355	*
6356	* @remarks No-long-jump zone!!!
6357	*/
6358	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6359	{
6360	int rc = VINF_SUCCESS;
6361	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6362	{
6363	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP));
6364	uint64_t u64Val = 0;
6365	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6366	AssertRCReturn(rc, rc);
6367
6368	pMixedCtx->rsp = u64Val;
6369	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6370	}
6371	return rc;
6372	}
6373
6374
6375	/**
6376	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6377	*
6378	* @returns VBox status code.
6379	* @param pVCpu The cross context virtual CPU structure.
6380	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6381	* out-of-sync. Make sure to update the required fields
6382	* before using them.
6383	*
6384	* @remarks No-long-jump zone!!!
6385	*/
6386	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6387	{
6388	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6389	{
6390	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS));
6391	uint32_t uVal = 0;
6392	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6393	AssertRCReturn(rc, rc);
6394
6395	pMixedCtx->eflags.u32 = uVal;
6396	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6397	{
6398	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6399	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6400
6401	pMixedCtx->eflags.Bits.u1VM = 0;
6402	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6403	}
6404
6405	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6406	}
6407	return VINF_SUCCESS;
6408	}
6409
6410
6411	/**
6412	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6413	* guest-CPU context.
6414	*/
6415	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6416	{
6417	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6418	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6419	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6420	return rc;
6421	}
6422
6423
6424	/**
6425	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6426	* from the guest-state area in the VMCS.
6427	*
6428	* @param pVCpu The cross context virtual CPU structure.
6429	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6430	* out-of-sync. Make sure to update the required fields
6431	* before using them.
6432	*
6433	* @remarks No-long-jump zone!!!
6434	*/
6435	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6436	{
6437	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6438	{
6439	uint32_t uIntrState = 0;
6440	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6441	AssertRC(rc);
6442
6443	if (!uIntrState)
6444	{
6445	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6446	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6447
6448	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6449	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6450	}
6451	else
6452	{
6453	if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6454	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6455	{
6456	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6457	AssertRC(rc);
6458	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6459	AssertRC(rc);
6460
6461	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6462	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6463	}
6464	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6465	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6466
6467	if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6468	{
6469	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6470	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6471	}
6472	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6473	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6474	}
6475
6476	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6477	}
6478	}
6479
6480
6481	/**
6482	* Saves the guest's activity state.
6483	*
6484	* @returns VBox status code.
6485	* @param pVCpu The cross context virtual CPU structure.
6486	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6487	* out-of-sync. Make sure to update the required fields
6488	* before using them.
6489	*
6490	* @remarks No-long-jump zone!!!
6491	*/
6492	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6493	{
6494	NOREF(pMixedCtx);
6495	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6496	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6497	return VINF_SUCCESS;
6498	}
6499
6500
6501	/**
6502	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6503	* the current VMCS into the guest-CPU context.
6504	*
6505	* @returns VBox status code.
6506	* @param pVCpu The cross context virtual CPU structure.
6507	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6508	* out-of-sync. Make sure to update the required fields
6509	* before using them.
6510	*
6511	* @remarks No-long-jump zone!!!
6512	*/
6513	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6514	{
6515	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6516	{
6517	Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR));
6518	uint32_t u32Val = 0;
6519	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6520	pMixedCtx->SysEnter.cs = u32Val;
6521	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6522	}
6523
6524	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6525	{
6526	Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR));
6527	uint64_t u64Val = 0;
6528	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6529	pMixedCtx->SysEnter.eip = u64Val;
6530	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6531	}
6532	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6533	{
6534	Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR));
6535	uint64_t u64Val = 0;
6536	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6537	pMixedCtx->SysEnter.esp = u64Val;
6538	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6539	}
6540	return VINF_SUCCESS;
6541	}
6542
6543
6544	/**
6545	* Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6546	* the CPU back into the guest-CPU context.
6547	*
6548	* @returns VBox status code.
6549	* @param pVCpu The cross context virtual CPU structure.
6550	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6551	* out-of-sync. Make sure to update the required fields
6552	* before using them.
6553	*
6554	* @remarks No-long-jump zone!!!
6555	*/
6556	static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6557	{
6558	/* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6559	VMMRZCallRing3Disable(pVCpu);
6560	HM_DISABLE_PREEMPT();
6561
6562	/* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6563	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6564	{
6565	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS));
6566	hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6567	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6568	}
6569
6570	HM_RESTORE_PREEMPT();
6571	VMMRZCallRing3Enable(pVCpu);
6572
6573	return VINF_SUCCESS;
6574	}
6575
6576
6577	/**
6578	* Saves the auto load/store'd guest MSRs from the current VMCS into
6579	* the guest-CPU context.
6580	*
6581	* @returns VBox status code.
6582	* @param pVCpu The cross context virtual CPU structure.
6583	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6584	* out-of-sync. Make sure to update the required fields
6585	* before using them.
6586	*
6587	* @remarks No-long-jump zone!!!
6588	*/
6589	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6590	{
6591	if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6592	return VINF_SUCCESS;
6593
6594	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS));
6595	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6596	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6597	Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6598	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6599	{
6600	switch (pMsr->u32Msr)
6601	{
6602	case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6603	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6604	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6605	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6606	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6607	case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6608	break;
6609
6610	default:
6611	{
6612	AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6613	pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6614	return VERR_HM_UNEXPECTED_LD_ST_MSR;
6615	}
6616	}
6617	}
6618
6619	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6620	return VINF_SUCCESS;
6621	}
6622
6623
6624	/**
6625	* Saves the guest control registers from the current VMCS into the guest-CPU
6626	* context.
6627	*
6628	* @returns VBox status code.
6629	* @param pVCpu The cross context virtual CPU structure.
6630	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6631	* out-of-sync. Make sure to update the required fields
6632	* before using them.
6633	*
6634	* @remarks No-long-jump zone!!!
6635	*/
6636	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6637	{
6638	/* Guest CR0. Guest FPU. */
6639	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6640	AssertRCReturn(rc, rc);
6641
6642	/* Guest CR4. */
6643	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6644	AssertRCReturn(rc, rc);
6645
6646	/* Guest CR2 - updated always during the world-switch or in #PF. */
6647	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6648	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6649	{
6650	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3));
6651	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6652	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6653
6654	PVM pVM = pVCpu->CTX_SUFF(pVM);
6655	if ( pVM->hm.s.vmx.fUnrestrictedGuest
6656	\|\| ( pVM->hm.s.fNestedPaging
6657	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6658	{
6659	uint64_t u64Val = 0;
6660	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6661	if (pMixedCtx->cr3 != u64Val)
6662	{
6663	CPUMSetGuestCR3(pVCpu, u64Val);
6664	if (VMMRZCallRing3IsEnabled(pVCpu))
6665	{
6666	PGMUpdateCR3(pVCpu, u64Val);
6667	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6668	}
6669	else
6670	{
6671	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6672	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6673	}
6674	}
6675
6676	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6677	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6678	{
6679	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6680	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6681	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6682	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6683	AssertRCReturn(rc, rc);
6684
6685	if (VMMRZCallRing3IsEnabled(pVCpu))
6686	{
6687	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6688	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6689	}
6690	else
6691	{
6692	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6693	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6694	}
6695	}
6696	}
6697
6698	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6699	}
6700
6701	/*
6702	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6703	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6704	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6705	*
6706	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6707	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6708	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6709	* -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6710	*
6711	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6712	*/
6713	if (VMMRZCallRing3IsEnabled(pVCpu))
6714	{
6715	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6716	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6717
6718	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6719	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6720
6721	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6722	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6723	}
6724
6725	return rc;
6726	}
6727
6728
6729	/**
6730	* Reads a guest segment register from the current VMCS into the guest-CPU
6731	* context.
6732	*
6733	* @returns VBox status code.
6734	* @param pVCpu The cross context virtual CPU structure.
6735	* @param idxSel Index of the selector in the VMCS.
6736	* @param idxLimit Index of the segment limit in the VMCS.
6737	* @param idxBase Index of the segment base in the VMCS.
6738	* @param idxAccess Index of the access rights of the segment in the VMCS.
6739	* @param pSelReg Pointer to the segment selector.
6740	*
6741	* @remarks No-long-jump zone!!!
6742	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6743	* macro as that takes care of whether to read from the VMCS cache or
6744	* not.
6745	*/
6746	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6747	PCPUMSELREG pSelReg)
6748	{
6749	NOREF(pVCpu);
6750
6751	uint32_t u32Val = 0;
6752	int rc = VMXReadVmcs32(idxSel, &u32Val);
6753	AssertRCReturn(rc, rc);
6754	pSelReg->Sel = (uint16_t)u32Val;
6755	pSelReg->ValidSel = (uint16_t)u32Val;
6756	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6757
6758	rc = VMXReadVmcs32(idxLimit, &u32Val);
6759	AssertRCReturn(rc, rc);
6760	pSelReg->u32Limit = u32Val;
6761
6762	uint64_t u64Val = 0;
6763	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6764	AssertRCReturn(rc, rc);
6765	pSelReg->u64Base = u64Val;
6766
6767	rc = VMXReadVmcs32(idxAccess, &u32Val);
6768	AssertRCReturn(rc, rc);
6769	pSelReg->Attr.u = u32Val;
6770
6771	/*
6772	* If VT-x marks the segment as unusable, most other bits remain undefined:
6773	* - For CS the L, D and G bits have meaning.
6774	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6775	* - For the remaining data segments no bits are defined.
6776	*
6777	* The present bit and the unusable bit has been observed to be set at the
6778	* same time (the selector was supposed to be invalid as we started executing
6779	* a V8086 interrupt in ring-0).
6780	*
6781	* What should be important for the rest of the VBox code, is that the P bit is
6782	* cleared. Some of the other VBox code recognizes the unusable bit, but
6783	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
6784	* safe side here, we'll strip off P and other bits we don't care about. If
6785	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6786	*
6787	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6788	*/
6789	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6790	{
6791	Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6792
6793	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6794	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
6795	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
6796
6797	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6798	#ifdef DEBUG_bird
6799	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6800	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6801	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6802	#endif
6803	}
6804	return VINF_SUCCESS;
6805	}
6806
6807
6808	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6809	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6810	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6811	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6812	#else
6813	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6814	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6815	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6816	#endif
6817
6818
6819	/**
6820	* Saves the guest segment registers from the current VMCS into the guest-CPU
6821	* context.
6822	*
6823	* @returns VBox status code.
6824	* @param pVCpu The cross context virtual CPU structure.
6825	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6826	* out-of-sync. Make sure to update the required fields
6827	* before using them.
6828	*
6829	* @remarks No-long-jump zone!!!
6830	*/
6831	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6832	{
6833	/* Guest segment registers. */
6834	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6835	{
6836	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS));
6837	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6838	AssertRCReturn(rc, rc);
6839
6840	rc = VMXLOCAL_READ_SEG(CS, cs);
6841	rc \|= VMXLOCAL_READ_SEG(SS, ss);
6842	rc \|= VMXLOCAL_READ_SEG(DS, ds);
6843	rc \|= VMXLOCAL_READ_SEG(ES, es);
6844	rc \|= VMXLOCAL_READ_SEG(FS, fs);
6845	rc \|= VMXLOCAL_READ_SEG(GS, gs);
6846	AssertRCReturn(rc, rc);
6847
6848	/* Restore segment attributes for real-on-v86 mode hack. */
6849	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6850	{
6851	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6852	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6853	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6854	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6855	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6856	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6857	}
6858	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6859	}
6860
6861	return VINF_SUCCESS;
6862	}
6863
6864
6865	/**
6866	* Saves the guest descriptor table registers and task register from the current
6867	* VMCS into the guest-CPU context.
6868	*
6869	* @returns VBox status code.
6870	* @param pVCpu The cross context virtual CPU structure.
6871	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6872	* out-of-sync. Make sure to update the required fields
6873	* before using them.
6874	*
6875	* @remarks No-long-jump zone!!!
6876	*/
6877	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6878	{
6879	int rc = VINF_SUCCESS;
6880
6881	/* Guest LDTR. */
6882	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6883	{
6884	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR));
6885	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6886	AssertRCReturn(rc, rc);
6887	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6888	}
6889
6890	/* Guest GDTR. */
6891	uint64_t u64Val = 0;
6892	uint32_t u32Val = 0;
6893	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6894	{
6895	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR));
6896	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6897	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6898	pMixedCtx->gdtr.pGdt = u64Val;
6899	pMixedCtx->gdtr.cbGdt = u32Val;
6900	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6901	}
6902
6903	/* Guest IDTR. */
6904	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6905	{
6906	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR));
6907	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6908	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6909	pMixedCtx->idtr.pIdt = u64Val;
6910	pMixedCtx->idtr.cbIdt = u32Val;
6911	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6912	}
6913
6914	/* Guest TR. */
6915	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6916	{
6917	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR));
6918	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6919	AssertRCReturn(rc, rc);
6920
6921	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6922	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6923	{
6924	rc = VMXLOCAL_READ_SEG(TR, tr);
6925	AssertRCReturn(rc, rc);
6926	}
6927	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6928	}
6929	return rc;
6930	}
6931
6932	#undef VMXLOCAL_READ_SEG
6933
6934
6935	/**
6936	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6937	* context.
6938	*
6939	* @returns VBox status code.
6940	* @param pVCpu The cross context virtual CPU structure.
6941	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6942	* out-of-sync. Make sure to update the required fields
6943	* before using them.
6944	*
6945	* @remarks No-long-jump zone!!!
6946	*/
6947	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6948	{
6949	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DR7))
6950	{
6951	if (!pVCpu->hm.s.fUsingHyperDR7)
6952	{
6953	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6954	uint32_t u32Val;
6955	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6956	pMixedCtx->dr[7] = u32Val;
6957	}
6958
6959	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DR7);
6960	}
6961	return VINF_SUCCESS;
6962	}
6963
6964
6965	/**
6966	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
6967	*
6968	* @returns VBox status code.
6969	* @param pVCpu The cross context virtual CPU structure.
6970	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6971	* out-of-sync. Make sure to update the required fields
6972	* before using them.
6973	*
6974	* @remarks No-long-jump zone!!!
6975	*/
6976	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6977	{
6978	NOREF(pMixedCtx);
6979
6980	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6981	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6982	return VINF_SUCCESS;
6983	}
6984
6985
6986	/**
6987	* Saves the entire guest state from the currently active VMCS into the
6988	* guest-CPU context.
6989	*
6990	* This essentially VMREADs all guest-data.
6991	*
6992	* @returns VBox status code.
6993	* @param pVCpu The cross context virtual CPU structure.
6994	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6995	* out-of-sync. Make sure to update the required fields
6996	* before using them.
6997	*/
6998	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6999	{
7000	Assert(pVCpu);
7001	Assert(pMixedCtx);
7002
7003	if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
7004	return VINF_SUCCESS;
7005
7006	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
7007	again on the ring-3 callback path, there is no real need to. */
7008	if (VMMRZCallRing3IsEnabled(pVCpu))
7009	VMMR0LogFlushDisable(pVCpu);
7010	else
7011	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7012	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
7013
7014	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7015	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7016
7017	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7018	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7019
7020	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7021	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7022
7023	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7024	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7025
7026	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
7027	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7028
7029	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
7030	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7031
7032	rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7033	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7034
7035	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7036	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7037
7038	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
7039	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7040
7041	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
7042	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7043
7044	AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
7045	("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
7046	HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
7047
7048	if (VMMRZCallRing3IsEnabled(pVCpu))
7049	VMMR0LogFlushEnable(pVCpu);
7050
7051	return VINF_SUCCESS;
7052	}
7053
7054
7055	/**
7056	* Saves basic guest registers needed for IEM instruction execution.
7057	*
7058	* @returns VBox status code (OR-able).
7059	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
7060	* @param pMixedCtx Pointer to the CPU context of the guest.
7061	* @param fMemory Whether the instruction being executed operates on
7062	* memory or not. Only CR0 is synced up if clear.
7063	* @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
7064	*/
7065	static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
7066	{
7067	/*
7068	* We assume all general purpose registers other than RSP are available.
7069	*
7070	* - RIP is a must, as it will be incremented or otherwise changed.
7071	* - RFLAGS are always required to figure the CPL.
7072	* - RSP isn't always required, however it's a GPR, so frequently required.
7073	* - SS and CS are the only segment register needed if IEM doesn't do memory
7074	* access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
7075	* - CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
7076	* be required for memory accesses.
7077	*
7078	* Note! Before IEM dispatches an exception, it will call us to sync in everything.
7079	*/
7080	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7081	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7082	if (fNeedRsp)
7083	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
7084	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7085	if (!fMemory)
7086	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7087	else
7088	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7089	AssertRCReturn(rc, rc);
7090	return rc;
7091	}
7092
7093
7094	/**
7095	* Ensures that we've got a complete basic guest-context.
7096	*
7097	* This excludes the FPU, SSE, AVX, and similar extended state. The interface
7098	* is for the interpreter.
7099	*
7100	* @returns VBox status code.
7101	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
7102	* @param pMixedCtx Pointer to the guest-CPU context which may have data
7103	* needing to be synced in.
7104	* @thread EMT(pVCpu)
7105	*/
7106	VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7107	{
7108	/* Note! Since this is only applicable to VT-x, the implementation is placed
7109	in the VT-x part of the sources instead of the generic stuff. */
7110	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
7111	{
7112	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7113	/*
7114	* For now, imply that the caller might change everything too. Do this after
7115	* saving the guest state so as to not trigger assertions.
7116	*/
7117	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7118	return rc;
7119	}
7120	return VINF_SUCCESS;
7121	}
7122
7123
7124	/**
7125	* Check per-VM and per-VCPU force flag actions that require us to go back to
7126	* ring-3 for one reason or another.
7127	*
7128	* @returns Strict VBox status code (i.e. informational status codes too)
7129	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7130	* ring-3.
7131	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7132	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7133	* interrupts)
7134	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7135	* all EMTs to be in ring-3.
7136	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7137	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7138	* to the EM loop.
7139	*
7140	* @param pVM The cross context VM structure.
7141	* @param pVCpu The cross context virtual CPU structure.
7142	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7143	* out-of-sync. Make sure to update the required fields
7144	* before using them.
7145	* @param fStepping Running in hmR0VmxRunGuestCodeStep().
7146	*/
7147	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
7148	{
7149	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7150
7151	/*
7152	* Anything pending? Should be more likely than not if we're doing a good job.
7153	*/
7154	if ( !fStepping
7155	? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7156	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7157	: !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7158	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7159	return VINF_SUCCESS;
7160
7161	/* We need the control registers now, make sure the guest-CPU context is updated. */
7162	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7163	AssertRCReturn(rc3, rc3);
7164
7165	/* Pending HM CR3 sync. */
7166	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7167	{
7168	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
7169	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
7170	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
7171	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7172	}
7173
7174	/* Pending HM PAE PDPEs. */
7175	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7176	{
7177	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7178	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7179	}
7180
7181	/* Pending PGM C3 sync. */
7182	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7183	{
7184	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
7185	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7186	if (rcStrict2 != VINF_SUCCESS)
7187	{
7188	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
7189	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
7190	return rcStrict2;
7191	}
7192	}
7193
7194	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7195	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
7196	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7197	{
7198	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7199	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
7200	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
7201	return rc2;
7202	}
7203
7204	/* Pending VM request packets, such as hardware interrupts. */
7205	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
7206	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
7207	{
7208	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
7209	return VINF_EM_PENDING_REQUEST;
7210	}
7211
7212	/* Pending PGM pool flushes. */
7213	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7214	{
7215	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
7216	return VINF_PGM_POOL_FLUSH_PENDING;
7217	}
7218
7219	/* Pending DMA requests. */
7220	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
7221	{
7222	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
7223	return VINF_EM_RAW_TO_R3;
7224	}
7225
7226	return VINF_SUCCESS;
7227	}
7228
7229
7230	/**
7231	* Converts any TRPM trap into a pending HM event. This is typically used when
7232	* entering from ring-3 (not longjmp returns).
7233	*
7234	* @param pVCpu The cross context virtual CPU structure.
7235	*/
7236	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7237	{
7238	Assert(TRPMHasTrap(pVCpu));
7239	Assert(!pVCpu->hm.s.Event.fPending);
7240
7241	uint8_t uVector;
7242	TRPMEVENT enmTrpmEvent;
7243	RTGCUINT uErrCode;
7244	RTGCUINTPTR GCPtrFaultAddress;
7245	uint8_t cbInstr;
7246
7247	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7248	AssertRC(rc);
7249
7250	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7251	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7252	if (enmTrpmEvent == TRPM_TRAP)
7253	{
7254	switch (uVector)
7255	{
7256	case X86_XCPT_NMI:
7257	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7258	break;
7259
7260	case X86_XCPT_BP:
7261	case X86_XCPT_OF:
7262	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7263	break;
7264
7265	case X86_XCPT_PF:
7266	case X86_XCPT_DF:
7267	case X86_XCPT_TS:
7268	case X86_XCPT_NP:
7269	case X86_XCPT_SS:
7270	case X86_XCPT_GP:
7271	case X86_XCPT_AC:
7272	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7273	RT_FALL_THRU();
7274	default:
7275	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7276	break;
7277	}
7278	}
7279	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7280	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7281	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7282	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7283	else
7284	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7285
7286	rc = TRPMResetTrap(pVCpu);
7287	AssertRC(rc);
7288	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7289	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7290
7291	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7292	}
7293
7294
7295	/**
7296	* Converts the pending HM event into a TRPM trap.
7297	*
7298	* @param pVCpu The cross context virtual CPU structure.
7299	*/
7300	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7301	{
7302	Assert(pVCpu->hm.s.Event.fPending);
7303
7304	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7305	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7306	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
7307	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7308
7309	/* If a trap was already pending, we did something wrong! */
7310	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7311
7312	TRPMEVENT enmTrapType;
7313	switch (uVectorType)
7314	{
7315	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7316	enmTrapType = TRPM_HARDWARE_INT;
7317	break;
7318
7319	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7320	enmTrapType = TRPM_SOFTWARE_INT;
7321	break;
7322
7323	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7324	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7325	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7326	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7327	enmTrapType = TRPM_TRAP;
7328	break;
7329
7330	default:
7331	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7332	enmTrapType = TRPM_32BIT_HACK;
7333	break;
7334	}
7335
7336	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7337
7338	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7339	AssertRC(rc);
7340
7341	if (fErrorCodeValid)
7342	TRPMSetErrorCode(pVCpu, uErrorCode);
7343
7344	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7345	&& uVector == X86_XCPT_PF)
7346	{
7347	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7348	}
7349	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7350	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7351	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7352	{
7353	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7354	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
7355	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7356	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7357	}
7358
7359	/* Clear any pending events from the VMCS. */
7360	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
7361	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
7362
7363	/* We're now done converting the pending event. */
7364	pVCpu->hm.s.Event.fPending = false;
7365	}
7366
7367
7368	/**
7369	* Does the necessary state syncing before returning to ring-3 for any reason
7370	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7371	*
7372	* @returns VBox status code.
7373	* @param pVCpu The cross context virtual CPU structure.
7374	* @param pMixedCtx Pointer to the guest-CPU context. The data may
7375	* be out-of-sync. Make sure to update the required
7376	* fields before using them.
7377	* @param fSaveGuestState Whether to save the guest state or not.
7378	*
7379	* @remarks No-long-jmp zone!!!
7380	*/
7381	static int hmR0VmxLeave(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7382	{
7383	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7384	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7385
7386	RTCPUID idCpu = RTMpCpuId();
7387	Log4Func(("HostCpuId=%u\n", idCpu));
7388
7389	/*
7390	* !!! IMPORTANT !!!
7391	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7392	*/
7393
7394	/* Save the guest state if necessary. */
7395	if ( fSaveGuestState
7396	&& HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7397	{
7398	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7399	AssertRCReturn(rc, rc);
7400	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7401	}
7402
7403	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
7404	if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7405	{
7406	/* We shouldn't reload CR0 without saving it first. */
7407	if (!fSaveGuestState)
7408	{
7409	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7410	AssertRCReturn(rc, rc);
7411	}
7412	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7413	}
7414
7415	/* Restore host debug registers if necessary and resync on next R0 reentry. */
7416	#ifdef VBOX_STRICT
7417	if (CPUMIsHyperDebugStateActive(pVCpu))
7418	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7419	#endif
7420	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7421	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7422	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7423	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7424
7425	#if HC_ARCH_BITS == 64
7426	/* Restore host-state bits that VT-x only restores partially. */
7427	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7428	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7429	{
7430	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7431	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7432	}
7433	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7434	#endif
7435
7436	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7437	if (pVCpu->hm.s.vmx.fLazyMsrs)
7438	{
7439	/* We shouldn't reload the guest MSRs without saving it first. */
7440	if (!fSaveGuestState)
7441	{
7442	int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7443	AssertRCReturn(rc, rc);
7444	}
7445	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7446	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7447	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7448	}
7449
7450	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7451	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7452
7453	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7454	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7455	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7456	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7457	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7458	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7459	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7460	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7461
7462	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7463
7464	/** @todo This partially defeats the purpose of having preemption hooks.
7465	* The problem is, deregistering the hooks should be moved to a place that
7466	* lasts until the EMT is about to be destroyed not everytime while leaving HM
7467	* context.
7468	*/
7469	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7470	{
7471	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7472	AssertRCReturn(rc, rc);
7473
7474	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7475	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7476	}
7477	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7478	NOREF(idCpu);
7479
7480	return VINF_SUCCESS;
7481	}
7482
7483
7484	/**
7485	* Leaves the VT-x session.
7486	*
7487	* @returns VBox status code.
7488	* @param pVCpu The cross context virtual CPU structure.
7489	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7490	* out-of-sync. Make sure to update the required fields
7491	* before using them.
7492	*
7493	* @remarks No-long-jmp zone!!!
7494	*/
7495	DECLINLINE(int) hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7496	{
7497	HM_DISABLE_PREEMPT();
7498	HMVMX_ASSERT_CPU_SAFE();
7499	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7500	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7501
7502	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7503	and done this from the VMXR0ThreadCtxCallback(). */
7504	if (!pVCpu->hm.s.fLeaveDone)
7505	{
7506	int rc2 = hmR0VmxLeave(pVCpu, pMixedCtx, true /* fSaveGuestState */);
7507	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7508	pVCpu->hm.s.fLeaveDone = true;
7509	}
7510	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7511
7512	/*
7513	* !!! IMPORTANT !!!
7514	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7515	*/
7516
7517	/* Deregister hook now that we've left HM context before re-enabling preemption. */
7518	/** @todo Deregistering here means we need to VMCLEAR always
7519	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
7520	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7521	VMMR0ThreadCtxHookDisable(pVCpu);
7522
7523	/* Leave HM context. This takes care of local init (term). */
7524	int rc = HMR0LeaveCpu(pVCpu);
7525
7526	HM_RESTORE_PREEMPT();
7527	return rc;
7528	}
7529
7530
7531	/**
7532	* Does the necessary state syncing before doing a longjmp to ring-3.
7533	*
7534	* @returns VBox status code.
7535	* @param pVCpu The cross context virtual CPU structure.
7536	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7537	* out-of-sync. Make sure to update the required fields
7538	* before using them.
7539	*
7540	* @remarks No-long-jmp zone!!!
7541	*/
7542	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7543	{
7544	return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7545	}
7546
7547
7548	/**
7549	* Take necessary actions before going back to ring-3.
7550	*
7551	* An action requires us to go back to ring-3. This function does the necessary
7552	* steps before we can safely return to ring-3. This is not the same as longjmps
7553	* to ring-3, this is voluntary and prepares the guest so it may continue
7554	* executing outside HM (recompiler/IEM).
7555	*
7556	* @returns VBox status code.
7557	* @param pVM The cross context VM structure.
7558	* @param pVCpu The cross context virtual CPU structure.
7559	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7560	* out-of-sync. Make sure to update the required fields
7561	* before using them.
7562	* @param rcExit The reason for exiting to ring-3. Can be
7563	* VINF_VMM_UNKNOWN_RING3_CALL.
7564	*/
7565	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7566	{
7567	Assert(pVM);
7568	Assert(pVCpu);
7569	Assert(pMixedCtx);
7570	HMVMX_ASSERT_PREEMPT_SAFE();
7571
7572	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7573	{
7574	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7575	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
7576	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7577	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7578	}
7579
7580	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7581	VMMRZCallRing3Disable(pVCpu);
7582	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7583
7584	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7585	if (pVCpu->hm.s.Event.fPending)
7586	{
7587	hmR0VmxPendingEventToTrpmTrap(pVCpu);
7588	Assert(!pVCpu->hm.s.Event.fPending);
7589	}
7590
7591	/* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7592	hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7593
7594	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7595	and if we're injecting an event we should have a TRPM trap pending. */
7596	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7597	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7598	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7599	#endif
7600
7601	/* Save guest state and restore host state bits. */
7602	int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7603	AssertRCReturn(rc, rc);
7604	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7605	/* Thread-context hooks are unregistered at this point!!! */
7606
7607	/* Sync recompiler state. */
7608	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7609	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7610	\| CPUM_CHANGED_LDTR
7611	\| CPUM_CHANGED_GDTR
7612	\| CPUM_CHANGED_IDTR
7613	\| CPUM_CHANGED_TR
7614	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
7615	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7616	if ( pVM->hm.s.fNestedPaging
7617	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
7618	{
7619	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7620	}
7621
7622	Assert(!pVCpu->hm.s.fClearTrapFlag);
7623
7624	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7625	if (rcExit != VINF_EM_RAW_INTERRUPT)
7626	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7627
7628	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7629
7630	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7631	VMMRZCallRing3RemoveNotification(pVCpu);
7632	VMMRZCallRing3Enable(pVCpu);
7633
7634	return rc;
7635	}
7636
7637
7638	/**
7639	* VMMRZCallRing3() callback wrapper which saves the guest state before we
7640	* longjump to ring-3 and possibly get preempted.
7641	*
7642	* @returns VBox status code.
7643	* @param pVCpu The cross context virtual CPU structure.
7644	* @param enmOperation The operation causing the ring-3 longjump.
7645	* @param pvUser Opaque pointer to the guest-CPU context. The data
7646	* may be out-of-sync. Make sure to update the required
7647	* fields before using them.
7648	*/
7649	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7650	{
7651	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7652	{
7653	/*
7654	* !!! IMPORTANT !!!
7655	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7656	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7657	*/
7658	VMMRZCallRing3RemoveNotification(pVCpu);
7659	VMMRZCallRing3Disable(pVCpu);
7660	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7661	RTThreadPreemptDisable(&PreemptState);
7662
7663	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7664	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7665
7666	#if HC_ARCH_BITS == 64
7667	/* Restore host-state bits that VT-x only restores partially. */
7668	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7669	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7670	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7671	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7672	#endif
7673	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7674	if (pVCpu->hm.s.vmx.fLazyMsrs)
7675	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7676
7677	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7678	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7679	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7680	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7681	{
7682	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7683	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7684	}
7685
7686	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7687	VMMR0ThreadCtxHookDisable(pVCpu);
7688	HMR0LeaveCpu(pVCpu);
7689	RTThreadPreemptRestore(&PreemptState);
7690	return VINF_SUCCESS;
7691	}
7692
7693	Assert(pVCpu);
7694	Assert(pvUser);
7695	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7696	HMVMX_ASSERT_PREEMPT_SAFE();
7697
7698	VMMRZCallRing3Disable(pVCpu);
7699	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7700
7701	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7702	enmOperation));
7703
7704	int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7705	AssertRCReturn(rc, rc);
7706
7707	VMMRZCallRing3Enable(pVCpu);
7708	return VINF_SUCCESS;
7709	}
7710
7711
7712	/**
7713	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7714	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7715	*
7716	* @param pVCpu The cross context virtual CPU structure.
7717	*/
7718	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7719	{
7720	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7721	{
7722	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7723	{
7724	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7725	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7726	AssertRC(rc);
7727	Log4(("Setup interrupt-window exiting\n"));
7728	}
7729	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7730	}
7731
7732
7733	/**
7734	* Clears the interrupt-window exiting control in the VMCS.
7735	*
7736	* @param pVCpu The cross context virtual CPU structure.
7737	*/
7738	DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7739	{
7740	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7741	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7742	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7743	AssertRC(rc);
7744	Log4(("Cleared interrupt-window exiting\n"));
7745	}
7746
7747
7748	/**
7749	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7750	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
7751	*
7752	* @param pVCpu The cross context virtual CPU structure.
7753	*/
7754	DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7755	{
7756	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7757	{
7758	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7759	{
7760	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7761	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7762	AssertRC(rc);
7763	Log4(("Setup NMI-window exiting\n"));
7764	}
7765	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7766	}
7767
7768
7769	/**
7770	* Clears the NMI-window exiting control in the VMCS.
7771	*
7772	* @param pVCpu The cross context virtual CPU structure.
7773	*/
7774	DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7775	{
7776	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7777	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7778	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7779	AssertRC(rc);
7780	Log4(("Cleared NMI-window exiting\n"));
7781	}
7782
7783
7784	/**
7785	* Evaluates the event to be delivered to the guest and sets it as the pending
7786	* event.
7787	*
7788	* @returns The VT-x guest-interruptibility state.
7789	* @param pVCpu The cross context virtual CPU structure.
7790	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7791	* out-of-sync. Make sure to update the required fields
7792	* before using them.
7793	*/
7794	static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7795	{
7796	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7797	uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7798	bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7799	bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7800	bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7801
7802	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7803	Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7804	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7805	Assert(!TRPMHasTrap(pVCpu));
7806
7807	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7808	APICUpdatePendingInterrupts(pVCpu);
7809
7810	/*
7811	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7812	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7813	*/
7814	/** @todo SMI. SMIs take priority over NMIs. */
7815	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7816	{
7817	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7818	if ( !pVCpu->hm.s.Event.fPending
7819	&& !fBlockNmi
7820	&& !fBlockSti
7821	&& !fBlockMovSS)
7822	{
7823	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7824	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7825	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7826
7827	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7828	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7829	}
7830	else
7831	hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7832	}
7833	/*
7834	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7835	* a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7836	*/
7837	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
7838	&& !pVCpu->hm.s.fSingleInstruction)
7839	{
7840	Assert(!DBGFIsStepping(pVCpu));
7841	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7842	AssertRC(rc);
7843	bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7844	if ( !pVCpu->hm.s.Event.fPending
7845	&& !fBlockInt
7846	&& !fBlockSti
7847	&& !fBlockMovSS)
7848	{
7849	uint8_t u8Interrupt;
7850	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7851	if (RT_SUCCESS(rc))
7852	{
7853	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7854	uint32_t u32IntInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7855	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7856
7857	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
7858	}
7859	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7860	{
7861	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7862	hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7863	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7864
7865	/*
7866	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7867	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7868	* need to re-set this force-flag here.
7869	*/
7870	}
7871	else
7872	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7873	}
7874	else
7875	hmR0VmxSetIntWindowExitVmcs(pVCpu);
7876	}
7877
7878	return uIntrState;
7879	}
7880
7881
7882	/**
7883	* Sets a pending-debug exception to be delivered to the guest if the guest is
7884	* single-stepping in the VMCS.
7885	*
7886	* @param pVCpu The cross context virtual CPU structure.
7887	*/
7888	DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7889	{
7890	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7891	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7892	AssertRC(rc);
7893	}
7894
7895
7896	/**
7897	* Injects any pending events into the guest if the guest is in a state to
7898	* receive them.
7899	*
7900	* @returns Strict VBox status code (i.e. informational status codes too).
7901	* @param pVCpu The cross context virtual CPU structure.
7902	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7903	* out-of-sync. Make sure to update the required fields
7904	* before using them.
7905	* @param uIntrState The VT-x guest-interruptibility state.
7906	* @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7907	* return VINF_EM_DBG_STEPPED if the event was
7908	* dispatched directly.
7909	*/
7910	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
7911	{
7912	HMVMX_ASSERT_PREEMPT_SAFE();
7913	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7914
7915	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7916	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7917
7918	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7919	Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7920	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7921	Assert(!TRPMHasTrap(pVCpu));
7922
7923	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7924	if (pVCpu->hm.s.Event.fPending)
7925	{
7926	/*
7927	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7928	* pending even while injecting an event and in this case, we want a VM-exit as soon as
7929	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
7930	*
7931	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7932	*/
7933	uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7934	#ifdef VBOX_STRICT
7935	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7936	{
7937	bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7938	Assert(!fBlockInt);
7939	Assert(!fBlockSti);
7940	Assert(!fBlockMovSS);
7941	}
7942	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7943	{
7944	bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7945	Assert(!fBlockSti);
7946	Assert(!fBlockMovSS);
7947	Assert(!fBlockNmi);
7948	}
7949	#endif
7950	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7951	(uint8_t)uIntType));
7952	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7953	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7954	fStepping, &uIntrState);
7955	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7956
7957	/* Update the interruptibility-state as it could have been changed by
7958	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7959	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7960	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7961
7962	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7963	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7964	else
7965	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7966	}
7967
7968	/* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7969	if ( fBlockSti
7970	\|\| fBlockMovSS)
7971	{
7972	if (!pVCpu->hm.s.fSingleInstruction)
7973	{
7974	/*
7975	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7976	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7977	* See Intel spec. 27.3.4 "Saving Non-Register State".
7978	*/
7979	Assert(!DBGFIsStepping(pVCpu));
7980	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7981	AssertRCReturn(rc2, rc2);
7982	if (pMixedCtx->eflags.Bits.u1TF)
7983	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7984	}
7985	else if (pMixedCtx->eflags.Bits.u1TF)
7986	{
7987	/*
7988	* We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7989	* BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7990	*/
7991	Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7992	uIntrState = 0;
7993	}
7994	}
7995
7996	/*
7997	* There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7998	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7999	*/
8000	int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
8001	AssertRC(rc2);
8002
8003	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8004	NOREF(fBlockMovSS); NOREF(fBlockSti);
8005	return rcStrict;
8006	}
8007
8008
8009	/**
8010	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
8011	*
8012	* @param pVCpu The cross context virtual CPU structure.
8013	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8014	* out-of-sync. Make sure to update the required fields
8015	* before using them.
8016	*/
8017	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8018	{
8019	NOREF(pMixedCtx);
8020	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8021	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
8022	}
8023
8024
8025	/**
8026	* Injects a double-fault (\#DF) exception into the VM.
8027	*
8028	* @returns Strict VBox status code (i.e. informational status codes too).
8029	* @param pVCpu The cross context virtual CPU structure.
8030	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8031	* out-of-sync. Make sure to update the required fields
8032	* before using them.
8033	* @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
8034	* and should return VINF_EM_DBG_STEPPED if the event
8035	* is injected directly (register modified by us, not
8036	* by hardware on VM-entry).
8037	* @param puIntrState Pointer to the current guest interruptibility-state.
8038	* This interruptibility-state will be updated if
8039	* necessary. This cannot not be NULL.
8040	*/
8041	DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
8042	{
8043	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8044	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8045	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8046	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
8047	fStepping, puIntrState);
8048	}
8049
8050
8051	/**
8052	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
8053	*
8054	* @param pVCpu The cross context virtual CPU structure.
8055	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8056	* out-of-sync. Make sure to update the required fields
8057	* before using them.
8058	*/
8059	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8060	{
8061	NOREF(pMixedCtx);
8062	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8063	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8064	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
8065	}
8066
8067
8068	/**
8069	* Sets an overflow (\#OF) exception as pending-for-injection into the VM.
8070	*
8071	* @param pVCpu The cross context virtual CPU structure.
8072	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8073	* out-of-sync. Make sure to update the required fields
8074	* before using them.
8075	* @param cbInstr The value of RIP that is to be pushed on the guest
8076	* stack.
8077	*/
8078	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
8079	{
8080	NOREF(pMixedCtx);
8081	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8082	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8083	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
8084	}
8085
8086
8087	/**
8088	* Injects a general-protection (\#GP) fault into the VM.
8089	*
8090	* @returns Strict VBox status code (i.e. informational status codes too).
8091	* @param pVCpu The cross context virtual CPU structure.
8092	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8093	* out-of-sync. Make sure to update the required fields
8094	* before using them.
8095	* @param fErrorCodeValid Whether the error code is valid (depends on the CPU
8096	* mode, i.e. in real-mode it's not valid).
8097	* @param u32ErrorCode The error code associated with the \#GP.
8098	* @param fStepping Whether we're running in
8099	* hmR0VmxRunGuestCodeStep() and should return
8100	* VINF_EM_DBG_STEPPED if the event is injected
8101	* directly (register modified by us, not by
8102	* hardware on VM-entry).
8103	* @param puIntrState Pointer to the current guest interruptibility-state.
8104	* This interruptibility-state will be updated if
8105	* necessary. This cannot not be NULL.
8106	*/
8107	DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
8108	bool fStepping, uint32_t *puIntrState)
8109	{
8110	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8111	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8112	if (fErrorCodeValid)
8113	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8114	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
8115	fStepping, puIntrState);
8116	}
8117
8118
8119	#if 0 /* unused */
8120	/**
8121	* Sets a general-protection (\#GP) exception as pending-for-injection into the
8122	* VM.
8123	*
8124	* @param pVCpu The cross context virtual CPU structure.
8125	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8126	* out-of-sync. Make sure to update the required fields
8127	* before using them.
8128	* @param u32ErrorCode The error code associated with the \#GP.
8129	*/
8130	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
8131	{
8132	NOREF(pMixedCtx);
8133	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8134	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8135	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8136	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */);
8137	}
8138	#endif /* unused */
8139
8140
8141	/**
8142	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
8143	*
8144	* @param pVCpu The cross context virtual CPU structure.
8145	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8146	* out-of-sync. Make sure to update the required fields
8147	* before using them.
8148	* @param uVector The software interrupt vector number.
8149	* @param cbInstr The value of RIP that is to be pushed on the guest
8150	* stack.
8151	*/
8152	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
8153	{
8154	NOREF(pMixedCtx);
8155	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8156	if ( uVector == X86_XCPT_BP
8157	\|\| uVector == X86_XCPT_OF)
8158	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8159	else
8160	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8161	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
8162	}
8163
8164
8165	/**
8166	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8167	* stack.
8168	*
8169	* @returns Strict VBox status code (i.e. informational status codes too).
8170	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8171	* @param pVM The cross context VM structure.
8172	* @param pMixedCtx Pointer to the guest-CPU context.
8173	* @param uValue The value to push to the guest stack.
8174	*/
8175	DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
8176	{
8177	/*
8178	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8179	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8180	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8181	*/
8182	if (pMixedCtx->sp == 1)
8183	return VINF_EM_RESET;
8184	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8185	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
8186	AssertRC(rc);
8187	return rc;
8188	}
8189
8190
8191	/**
8192	* Injects an event into the guest upon VM-entry by updating the relevant fields
8193	* in the VM-entry area in the VMCS.
8194	*
8195	* @returns Strict VBox status code (i.e. informational status codes too).
8196	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8197	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8198	*
8199	* @param pVCpu The cross context virtual CPU structure.
8200	* @param pMixedCtx Pointer to the guest-CPU context. The data may
8201	* be out-of-sync. Make sure to update the required
8202	* fields before using them.
8203	* @param u64IntInfo The VM-entry interruption-information field.
8204	* @param cbInstr The VM-entry instruction length in bytes (for
8205	* software interrupts, exceptions and privileged
8206	* software exceptions).
8207	* @param u32ErrCode The VM-entry exception error code.
8208	* @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
8209	* @param puIntrState Pointer to the current guest interruptibility-state.
8210	* This interruptibility-state will be updated if
8211	* necessary. This cannot not be NULL.
8212	* @param fStepping Whether we're running in
8213	* hmR0VmxRunGuestCodeStep() and should return
8214	* VINF_EM_DBG_STEPPED if the event is injected
8215	* directly (register modified by us, not by
8216	* hardware on VM-entry).
8217	*
8218	* @remarks Requires CR0!
8219	*/
8220	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
8221	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
8222	uint32_t *puIntrState)
8223	{
8224	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8225	AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
8226	Assert(puIntrState);
8227	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
8228
8229	uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
8230	uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
8231
8232	#ifdef VBOX_STRICT
8233	/*
8234	* Validate the error-code-valid bit for hardware exceptions.
8235	* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8236	*/
8237	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8238	&& !CPUMIsGuestInRealModeEx(pMixedCtx))
8239	{
8240	switch (uVector)
8241	{
8242	case X86_XCPT_PF:
8243	case X86_XCPT_DF:
8244	case X86_XCPT_TS:
8245	case X86_XCPT_NP:
8246	case X86_XCPT_SS:
8247	case X86_XCPT_GP:
8248	case X86_XCPT_AC:
8249	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
8250	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8251	RT_FALL_THRU();
8252	default:
8253	break;
8254	}
8255	}
8256	#endif
8257
8258	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8259	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8260	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
8261
8262	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8263
8264	/* We require CR0 to check if the guest is in real-mode. */
8265	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8266	AssertRCReturn(rc, rc);
8267
8268	/*
8269	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
8270	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
8271	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
8272	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8273	*/
8274	if (CPUMIsGuestInRealModeEx(pMixedCtx))
8275	{
8276	PVM pVM = pVCpu->CTX_SUFF(pVM);
8277	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
8278	{
8279	Assert(PDMVmmDevHeapIsEnabled(pVM));
8280	Assert(pVM->hm.s.vmx.pRealModeTSS);
8281
8282	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
8283	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8284	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
8285	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8286	AssertRCReturn(rc, rc);
8287	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
8288
8289	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8290	size_t const cbIdtEntry = sizeof(X86IDTR16);
8291	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
8292	{
8293	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8294	if (uVector == X86_XCPT_DF)
8295	return VINF_EM_RESET;
8296
8297	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
8298	if (uVector == X86_XCPT_GP)
8299	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
8300
8301	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
8302	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
8303	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */,
8304	fStepping, puIntrState);
8305	}
8306
8307	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8308	uint16_t uGuestIp = pMixedCtx->ip;
8309	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
8310	{
8311	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8312	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8313	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8314	}
8315	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
8316	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8317
8318	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8319	X86IDTR16 IdtEntry;
8320	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
8321	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8322	AssertRCReturn(rc, rc);
8323
8324	/* Construct the stack frame for the interrupt/exception handler. */
8325	VBOXSTRICTRC rcStrict;
8326	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
8327	if (rcStrict == VINF_SUCCESS)
8328	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
8329	if (rcStrict == VINF_SUCCESS)
8330	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
8331
8332	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8333	if (rcStrict == VINF_SUCCESS)
8334	{
8335	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8336	pMixedCtx->rip = IdtEntry.offSel;
8337	pMixedCtx->cs.Sel = IdtEntry.uSel;
8338	pMixedCtx->cs.ValidSel = IdtEntry.uSel;
8339	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8340	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8341	&& uVector == X86_XCPT_PF)
8342	pMixedCtx->cr2 = GCPtrFaultAddress;
8343
8344	/* If any other guest-state bits are changed here, make sure to update
8345	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
8346	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
8347	\| HM_CHANGED_GUEST_RIP
8348	\| HM_CHANGED_GUEST_RFLAGS
8349	\| HM_CHANGED_GUEST_RSP);
8350
8351	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8352	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8353	{
8354	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8355	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8356	Log4(("Clearing inhibition due to STI.\n"));
8357	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
8358	}
8359	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8360	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
8361
8362	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8363	it, if we are returning to ring-3 before executing guest code. */
8364	pVCpu->hm.s.Event.fPending = false;
8365
8366	/* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
8367	if (fStepping)
8368	rcStrict = VINF_EM_DBG_STEPPED;
8369	}
8370	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8371	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8372	return rcStrict;
8373	}
8374
8375	/*
8376	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
8377	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8378	*/
8379	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8380	}
8381
8382	/* Validate. */
8383	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8384	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8385	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8386
8387	/* Inject. */
8388	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8389	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8390	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8391	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8392
8393	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8394	&& uVector == X86_XCPT_PF)
8395	pMixedCtx->cr2 = GCPtrFaultAddress;
8396
8397	Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8398	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8399
8400	AssertRCReturn(rc, rc);
8401	return VINF_SUCCESS;
8402	}
8403
8404
8405	/**
8406	* Clears the interrupt-window exiting control in the VMCS and if necessary
8407	* clears the current event in the VMCS as well.
8408	*
8409	* @returns VBox status code.
8410	* @param pVCpu The cross context virtual CPU structure.
8411	*
8412	* @remarks Use this function only to clear events that have not yet been
8413	* delivered to the guest but are injected in the VMCS!
8414	* @remarks No-long-jump zone!!!
8415	*/
8416	static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8417	{
8418	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8419
8420	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8421	hmR0VmxClearIntWindowExitVmcs(pVCpu);
8422
8423	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8424	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8425	}
8426
8427
8428	/**
8429	* Enters the VT-x session.
8430	*
8431	* @returns VBox status code.
8432	* @param pVM The cross context VM structure.
8433	* @param pVCpu The cross context virtual CPU structure.
8434	* @param pCpu Pointer to the CPU info struct.
8435	*/
8436	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8437	{
8438	AssertPtr(pVM);
8439	AssertPtr(pVCpu);
8440	Assert(pVM->hm.s.vmx.fSupported);
8441	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8442	NOREF(pCpu); NOREF(pVM);
8443
8444	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8445	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
8446
8447	#ifdef VBOX_STRICT
8448	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8449	RTCCUINTREG uHostCR4 = ASMGetCR4();
8450	if (!(uHostCR4 & X86_CR4_VMXE))
8451	{
8452	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8453	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8454	}
8455	#endif
8456
8457	/*
8458	* Load the VCPU's VMCS as the current (and active) one.
8459	*/
8460	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8461	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8462	if (RT_FAILURE(rc))
8463	return rc;
8464
8465	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8466	pVCpu->hm.s.fLeaveDone = false;
8467	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8468
8469	return VINF_SUCCESS;
8470	}
8471
8472
8473	/**
8474	* The thread-context callback (only on platforms which support it).
8475	*
8476	* @param enmEvent The thread-context event.
8477	* @param pVCpu The cross context virtual CPU structure.
8478	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8479	* @thread EMT(pVCpu)
8480	*/
8481	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8482	{
8483	NOREF(fGlobalInit);
8484
8485	switch (enmEvent)
8486	{
8487	case RTTHREADCTXEVENT_OUT:
8488	{
8489	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8490	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8491	VMCPU_ASSERT_EMT(pVCpu);
8492
8493	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8494
8495	/* No longjmps (logger flushes, locks) in this fragile context. */
8496	VMMRZCallRing3Disable(pVCpu);
8497	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8498
8499	/*
8500	* Restore host-state (FPU, debug etc.)
8501	*/
8502	if (!pVCpu->hm.s.fLeaveDone)
8503	{
8504	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8505	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8506	hmR0VmxLeave(pVCpu, pMixedCtx, false /* fSaveGuestState */);
8507	pVCpu->hm.s.fLeaveDone = true;
8508	}
8509
8510	/* Leave HM context, takes care of local init (term). */
8511	int rc = HMR0LeaveCpu(pVCpu);
8512	AssertRC(rc); NOREF(rc);
8513
8514	/* Restore longjmp state. */
8515	VMMRZCallRing3Enable(pVCpu);
8516	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8517	break;
8518	}
8519
8520	case RTTHREADCTXEVENT_IN:
8521	{
8522	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8523	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8524	VMCPU_ASSERT_EMT(pVCpu);
8525
8526	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8527	VMMRZCallRing3Disable(pVCpu);
8528	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8529
8530	/* Initialize the bare minimum state required for HM. This takes care of
8531	initializing VT-x if necessary (onlined CPUs, local init etc.) */
8532	int rc = HMR0EnterCpu(pVCpu);
8533	AssertRC(rc);
8534	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
8535
8536	/* Load the active VMCS as the current one. */
8537	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8538	{
8539	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8540	AssertRC(rc); NOREF(rc);
8541	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8542	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8543	}
8544	pVCpu->hm.s.fLeaveDone = false;
8545
8546	/* Restore longjmp state. */
8547	VMMRZCallRing3Enable(pVCpu);
8548	break;
8549	}
8550
8551	default:
8552	break;
8553	}
8554	}
8555
8556
8557	/**
8558	* Saves the host state in the VMCS host-state.
8559	* Sets up the VM-exit MSR-load area.
8560	*
8561	* The CPU state will be loaded from these fields on every successful VM-exit.
8562	*
8563	* @returns VBox status code.
8564	* @param pVM The cross context VM structure.
8565	* @param pVCpu The cross context virtual CPU structure.
8566	*
8567	* @remarks No-long-jump zone!!!
8568	*/
8569	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8570	{
8571	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8572
8573	int rc = VINF_SUCCESS;
8574	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8575	{
8576	rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8577	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8578
8579	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8580	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8581
8582	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8583	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8584
8585	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8586	}
8587	return rc;
8588	}
8589
8590
8591	/**
8592	* Saves the host state in the VMCS host-state.
8593	*
8594	* @returns VBox status code.
8595	* @param pVM The cross context VM structure.
8596	* @param pVCpu The cross context virtual CPU structure.
8597	*
8598	* @remarks No-long-jump zone!!!
8599	*/
8600	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8601	{
8602	AssertPtr(pVM);
8603	AssertPtr(pVCpu);
8604
8605	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8606
8607	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8608	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8609	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8610	return hmR0VmxSaveHostState(pVM, pVCpu);
8611	}
8612
8613
8614	/**
8615	* Loads the guest state into the VMCS guest-state area.
8616	*
8617	* The will typically be done before VM-entry when the guest-CPU state and the
8618	* VMCS state may potentially be out of sync.
8619	*
8620	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8621	* VM-entry controls.
8622	* Sets up the appropriate VMX non-root function to execute guest code based on
8623	* the guest CPU mode.
8624	*
8625	* @returns VBox strict status code.
8626	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8627	* without unrestricted guest access and the VMMDev is not presently
8628	* mapped (e.g. EFI32).
8629	*
8630	* @param pVM The cross context VM structure.
8631	* @param pVCpu The cross context virtual CPU structure.
8632	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8633	* out-of-sync. Make sure to update the required fields
8634	* before using them.
8635	*
8636	* @remarks No-long-jump zone!!!
8637	*/
8638	static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8639	{
8640	AssertPtr(pVM);
8641	AssertPtr(pVCpu);
8642	AssertPtr(pMixedCtx);
8643	HMVMX_ASSERT_PREEMPT_SAFE();
8644
8645	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8646
8647	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8648
8649	/* Determine real-on-v86 mode. */
8650	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8651	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8652	&& CPUMIsGuestInRealModeEx(pMixedCtx))
8653	{
8654	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8655	}
8656
8657	/*
8658	* Load the guest-state into the VMCS.
8659	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8660	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8661	*/
8662	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8663	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8664
8665	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8666	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8667	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8668
8669	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8670	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8671	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8672
8673	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8674	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8675
8676	VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8677	if (rcStrict == VINF_SUCCESS)
8678	{ /* likely */ }
8679	else
8680	{
8681	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
8682	return rcStrict;
8683	}
8684
8685	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8686	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8687	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8688
8689	/* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8690	determine we don't have to swap EFER after all. */
8691	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8692	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8693
8694	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8695	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8696
8697	rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8698	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8699
8700	/*
8701	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8702	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8703	*/
8704	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8705	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8706
8707	/* Clear any unused and reserved bits. */
8708	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8709
8710	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8711	return rc;
8712	}
8713
8714
8715	/**
8716	* Loads the state shared between the host and guest into the VMCS.
8717	*
8718	* @param pVM The cross context VM structure.
8719	* @param pVCpu The cross context virtual CPU structure.
8720	* @param pCtx Pointer to the guest-CPU context.
8721	*
8722	* @remarks No-long-jump zone!!!
8723	*/
8724	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8725	{
8726	NOREF(pVM);
8727
8728	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8729	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8730
8731	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8732	{
8733	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8734	AssertRC(rc);
8735	}
8736
8737	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8738	{
8739	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8740	AssertRC(rc);
8741
8742	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8743	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8744	{
8745	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8746	AssertRC(rc);
8747	}
8748	}
8749
8750	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8751	{
8752	hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8753	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8754	}
8755
8756	/* Loading CR0, debug state might have changed intercepts, update VMCS. */
8757	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8758	{
8759	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8760	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8761	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8762	AssertRC(rc);
8763	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8764	}
8765
8766	AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8767	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8768	}
8769
8770
8771	/**
8772	* Worker for loading the guest-state bits in the inner VT-x execution loop.
8773	*
8774	* @returns Strict VBox status code (i.e. informational status codes too).
8775	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8776	* without unrestricted guest access and the VMMDev is not presently
8777	* mapped (e.g. EFI32).
8778	*
8779	* @param pVM The cross context VM structure.
8780	* @param pVCpu The cross context virtual CPU structure.
8781	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8782	* out-of-sync. Make sure to update the required fields
8783	* before using them.
8784	*
8785	* @remarks No-long-jump zone!!!
8786	*/
8787	static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8788	{
8789	HMVMX_ASSERT_PREEMPT_SAFE();
8790	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8791	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8792
8793	Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8794	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8795	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8796	#endif
8797
8798	/*
8799	* RIP is what changes the most often and hence if it's the only bit needing to be
8800	* updated, we shall handle it early for performance reasons.
8801	*/
8802	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8803	if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8804	{
8805	rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8806	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8807	{ /* likely */}
8808	else
8809	{
8810	AssertMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8811	VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8812	}
8813	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8814	}
8815	else if (HMCPU_CF_VALUE(pVCpu))
8816	{
8817	rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8818	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8819	{ /* likely */}
8820	else
8821	{
8822	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
8823	("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8824	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8825	return rcStrict;
8826	}
8827	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8828	}
8829
8830	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8831	AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8832	\|\| HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
8833	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8834	return rcStrict;
8835	}
8836
8837
8838	/**
8839	* Does the preparations before executing guest code in VT-x.
8840	*
8841	* This may cause longjmps to ring-3 and may even result in rescheduling to the
8842	* recompiler/IEM. We must be cautious what we do here regarding committing
8843	* guest-state information into the VMCS assuming we assuredly execute the
8844	* guest in VT-x mode.
8845	*
8846	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
8847	* the common-state (TRPM/forceflags), we must undo those changes so that the
8848	* recompiler/IEM can (and should) use them when it resumes guest execution.
8849	* Otherwise such operations must be done when we can no longer exit to ring-3.
8850	*
8851	* @returns Strict VBox status code (i.e. informational status codes too).
8852	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8853	* have been disabled.
8854	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8855	* double-fault into the guest.
8856	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8857	* dispatched directly.
8858	* @retval VINF_* scheduling changes, we have to go back to ring-3.
8859	*
8860	* @param pVM The cross context VM structure.
8861	* @param pVCpu The cross context virtual CPU structure.
8862	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8863	* out-of-sync. Make sure to update the required fields
8864	* before using them.
8865	* @param pVmxTransient Pointer to the VMX transient structure.
8866	* @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8867	* us ignore some of the reasons for returning to
8868	* ring-3, and return VINF_EM_DBG_STEPPED if event
8869	* dispatching took place.
8870	*/
8871	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8872	{
8873	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8874
8875	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8876	PGMRZDynMapFlushAutoSet(pVCpu);
8877	#endif
8878
8879	/* Check force flag actions that might require us to go back to ring-3. */
8880	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8881	if (rcStrict == VINF_SUCCESS)
8882	{ /* FFs doesn't get set all the time. */ }
8883	else
8884	return rcStrict;
8885
8886	#ifndef IEM_VERIFICATION_MODE_FULL
8887	/*
8888	* Setup the virtualized-APIC accesses.
8889	*
8890	* Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8891	* in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8892	*
8893	* This is the reason we do it here and not in hmR0VmxLoadGuestState().
8894	*/
8895	if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8896	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8897	&& PDMHasApic(pVM))
8898	{
8899	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8900	Assert(u64MsrApicBase);
8901	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8902
8903	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8904
8905	/* Unalias any existing mapping. */
8906	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8907	AssertRCReturn(rc, rc);
8908
8909	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8910	Log4(("hmR0VmxPreRunGuest: VCPU%u: Mapped HC APIC-access page at %#RGp\n", pVCpu->idCpu, GCPhysApicBase));
8911	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
8912	AssertRCReturn(rc, rc);
8913
8914	/* Update the per-VCPU cache of the APIC base MSR. */
8915	pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8916	}
8917	#endif /* !IEM_VERIFICATION_MODE_FULL */
8918
8919	if (TRPMHasTrap(pVCpu))
8920	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8921	uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8922
8923	/*
8924	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8925	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8926	*/
8927	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
8928	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8929	{ /* likely */ }
8930	else
8931	{
8932	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8933	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8934	return rcStrict;
8935	}
8936
8937	/*
8938	* No longjmps to ring-3 from this point on!!!
8939	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8940	* This also disables flushing of the R0-logger instance (if any).
8941	*/
8942	VMMRZCallRing3Disable(pVCpu);
8943
8944	/*
8945	* Load the guest state bits.
8946	*
8947	* We cannot perform longjmps while loading the guest state because we do not preserve the
8948	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
8949	* CPU migration.
8950	*
8951	* If we are injecting events to a real-on-v86 mode guest, we will have to update
8952	* RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8953	* Hence, loading of the guest state needs to be done -after- injection of events.
8954	*/
8955	rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8956	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8957	{ /* likely */ }
8958	else
8959	{
8960	VMMRZCallRing3Enable(pVCpu);
8961	return rcStrict;
8962	}
8963
8964	/*
8965	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8966	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8967	*
8968	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8969	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8970	*
8971	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8972	* executing guest code.
8973	*/
8974	pVmxTransient->fEFlags = ASMIntDisableFlags();
8975
8976	if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
8977	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8978	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8979	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
8980	{
8981	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8982	{
8983	pVCpu->hm.s.Event.fPending = false;
8984
8985	/*
8986	* We've injected any pending events. This is really the point of no return (to ring-3).
8987	*
8988	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
8989	* returns from this function, so don't enable them here.
8990	*/
8991	return VINF_SUCCESS;
8992	}
8993
8994	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8995	rcStrict = VINF_EM_RAW_INTERRUPT;
8996	}
8997	else
8998	{
8999	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
9000	rcStrict = VINF_EM_RAW_TO_R3;
9001	}
9002
9003	ASMSetFlags(pVmxTransient->fEFlags);
9004	VMMRZCallRing3Enable(pVCpu);
9005
9006	return rcStrict;
9007	}
9008
9009
9010	/**
9011	* Prepares to run guest code in VT-x and we've committed to doing so. This
9012	* means there is no backing out to ring-3 or anywhere else at this
9013	* point.
9014	*
9015	* @param pVM The cross context VM structure.
9016	* @param pVCpu The cross context virtual CPU structure.
9017	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
9018	* out-of-sync. Make sure to update the required fields
9019	* before using them.
9020	* @param pVmxTransient Pointer to the VMX transient structure.
9021	*
9022	* @remarks Called with preemption disabled.
9023	* @remarks No-long-jump zone!!!
9024	*/
9025	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9026	{
9027	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9028	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9029	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9030
9031	/*
9032	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
9033	*/
9034	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
9035	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
9036
9037	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
9038	if (!CPUMIsGuestFPUStateActive(pVCpu))
9039	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
9040	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
9041	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9042	#endif
9043
9044	if ( pVCpu->hm.s.fPreloadGuestFpu
9045	&& !CPUMIsGuestFPUStateActive(pVCpu))
9046	{
9047	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
9048	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
9049	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
9050	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9051	}
9052
9053	/*
9054	* Lazy-update of the host MSRs values in the auto-load/store MSR area.
9055	*/
9056	if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
9057	&& pVCpu->hm.s.vmx.cMsrs > 0)
9058	{
9059	hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
9060	}
9061
9062	/*
9063	* Load the host state bits as we may've been preempted (only happens when
9064	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
9065	* Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
9066	* if we change the switcher back to 32-bit, we must save the 32-bit host state here.
9067	* See @bugref{8432}.
9068	*/
9069	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
9070	{
9071	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
9072	AssertRC(rc);
9073	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
9074	}
9075	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
9076
9077	/*
9078	* Load the state shared between host and guest (FPU, debug, lazy MSRs).
9079	*/
9080	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
9081	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
9082	AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
9083
9084	/* Store status of the shared guest-host state at the time of VM-entry. */
9085	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
9086	if (CPUMIsGuestInLongModeEx(pMixedCtx))
9087	{
9088	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
9089	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
9090	}
9091	else
9092	#endif
9093	{
9094	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
9095	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
9096	}
9097	pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
9098
9099	/*
9100	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
9101	*/
9102	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9103	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
9104
9105	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
9106	RTCPUID idCurrentCpu = pCpu->idCpu;
9107	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
9108	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
9109	{
9110	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
9111	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
9112	}
9113
9114	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
9115	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
9116	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
9117	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
9118
9119	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
9120
9121	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
9122	to start executing. */
9123
9124	/*
9125	* Load the TSC_AUX MSR when we are not intercepting RDTSCP.
9126	*/
9127	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
9128	{
9129	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9130	{
9131	bool fMsrUpdated;
9132	int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9133	AssertRC(rc2);
9134	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
9135
9136	rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
9137	&fMsrUpdated);
9138	AssertRC(rc2);
9139	Assert(fMsrUpdated \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9140
9141	/* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
9142	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
9143	}
9144	else
9145	{
9146	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
9147	Assert(!pVCpu->hm.s.vmx.cMsrs \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9148	}
9149	}
9150
9151	#ifdef VBOX_STRICT
9152	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
9153	hmR0VmxCheckHostEferMsr(pVCpu);
9154	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
9155	#endif
9156	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
9157	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
9158	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
9159	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
9160	#endif
9161	}
9162
9163
9164	/**
9165	* Performs some essential restoration of state after running guest code in
9166	* VT-x.
9167	*
9168	* @param pVM The cross context VM structure.
9169	* @param pVCpu The cross context virtual CPU structure.
9170	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9171	* out-of-sync. Make sure to update the required fields
9172	* before using them.
9173	* @param pVmxTransient Pointer to the VMX transient structure.
9174	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
9175	*
9176	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
9177	*
9178	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
9179	* unconditionally when it is safe to do so.
9180	*/
9181	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
9182	{
9183	NOREF(pVM);
9184
9185	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9186
9187	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
9188	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
9189	HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
9190	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
9191	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
9192	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
9193
9194	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9195	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
9196
9197	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
9198	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
9199	Assert(!ASMIntAreEnabled());
9200	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
9201
9202	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
9203	if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVM, pVCpu))
9204	{
9205	hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9206	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9207	}
9208	#endif
9209
9210	#if HC_ARCH_BITS == 64
9211	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
9212	#endif
9213	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
9214	/* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
9215	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
9216	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
9217	#else
9218	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
9219	#endif
9220	#ifdef VBOX_STRICT
9221	hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
9222	#endif
9223	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
9224
9225	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
9226	uint32_t uExitReason;
9227	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
9228	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9229	AssertRC(rc);
9230	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
9231	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
9232
9233	if (rcVMRun == VINF_SUCCESS)
9234	{
9235	/*
9236	* Update the VM-exit history array here even if the VM-entry failed due to:
9237	* - Invalid guest state.
9238	* - MSR loading.
9239	* - Machine-check event.
9240	*
9241	* In any of the above cases we will still have a "valid" VM-exit reason
9242	* despite @a fVMEntryFailed being false.
9243	*
9244	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
9245	*/
9246	HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
9247
9248	if (!pVmxTransient->fVMEntryFailed)
9249	{
9250	/** @todo We can optimize this by only syncing with our force-flags when
9251	* really needed and keeping the VMCS state as it is for most
9252	* VM-exits. */
9253	/* Update the guest interruptibility-state from the VMCS. */
9254	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
9255
9256	/*
9257	* Allow longjmps to ring-3 -after- saving the guest-interruptibility state
9258	* as it's not part of hmR0VmxSaveGuestState() and thus would trigger an assertion
9259	* on the longjmp path to ring-3 while saving the (rest of) the guest state,
9260	* see @bugref{6208#c63}.
9261	*/
9262	VMMRZCallRing3Enable(pVCpu);
9263
9264	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
9265	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9266	AssertRC(rc);
9267	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
9268	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9269	AssertRC(rc);
9270	#endif
9271
9272	/*
9273	* Sync the TPR shadow with our APIC state.
9274	*/
9275	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9276	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
9277	{
9278	rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
9279	AssertRC(rc);
9280	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9281	}
9282
9283	return;
9284	}
9285	}
9286	else
9287	{
9288	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
9289	pVmxTransient->fVMEntryFailed));
9290	}
9291
9292	VMMRZCallRing3Enable(pVCpu);
9293	}
9294
9295
9296	/**
9297	* Runs the guest code using VT-x the normal way.
9298	*
9299	* @returns VBox status code.
9300	* @param pVM The cross context VM structure.
9301	* @param pVCpu The cross context virtual CPU structure.
9302	* @param pCtx Pointer to the guest-CPU context.
9303	*
9304	* @note Mostly the same as hmR0VmxRunGuestCodeStep().
9305	*/
9306	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9307	{
9308	VMXTRANSIENT VmxTransient;
9309	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9310	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9311	uint32_t cLoops = 0;
9312
9313	for (;; cLoops++)
9314	{
9315	Assert(!HMR0SuspendPending());
9316	HMVMX_ASSERT_CPU_SAFE();
9317
9318	/* Preparatory work for running guest code, this may force us to return
9319	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
9320	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9321	rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
9322	if (rcStrict != VINF_SUCCESS)
9323	break;
9324
9325	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9326	int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9327	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9328
9329	/* Restore any residual host-state and save any bits shared between host
9330	and guest into the guest-CPU state. Re-enables interrupts! */
9331	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
9332
9333	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9334	if (RT_SUCCESS(rcRun))
9335	{ /* very likely */ }
9336	else
9337	{
9338	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9339	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9340	return rcRun;
9341	}
9342
9343	/* Profile the VM-exit. */
9344	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9345	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9346	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9347	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9348	HMVMX_START_EXIT_DISPATCH_PROF();
9349
9350	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9351
9352	/* Handle the VM-exit. */
9353	#ifdef HMVMX_USE_FUNCTION_TABLE
9354	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
9355	#else
9356	rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
9357	#endif
9358	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9359	if (rcStrict == VINF_SUCCESS)
9360	{
9361	if (cLoops <= pVM->hm.s.cMaxResumeLoops)
9362	continue; /* likely */
9363	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9364	rcStrict = VINF_EM_RAW_INTERRUPT;
9365	}
9366	break;
9367	}
9368
9369	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9370	return rcStrict;
9371	}
9372
9373
9374
9375	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
9376	* probes.
9377	*
9378	* The following few functions and associated structure contains the bloat
9379	* necessary for providing detailed debug events and dtrace probes as well as
9380	* reliable host side single stepping. This works on the principle of
9381	* "subclassing" the normal execution loop and workers. We replace the loop
9382	* method completely and override selected helpers to add necessary adjustments
9383	* to their core operation.
9384	*
9385	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
9386	* any performance for debug and analysis features.
9387	*
9388	* @{
9389	*/
9390
9391	/**
9392	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
9393	* the debug run loop.
9394	*/
9395	typedef struct VMXRUNDBGSTATE
9396	{
9397	/** The RIP we started executing at. This is for detecting that we stepped. */
9398	uint64_t uRipStart;
9399	/** The CS we started executing with. */
9400	uint16_t uCsStart;
9401
9402	/** Whether we've actually modified the 1st execution control field. */
9403	bool fModifiedProcCtls : 1;
9404	/** Whether we've actually modified the 2nd execution control field. */
9405	bool fModifiedProcCtls2 : 1;
9406	/** Whether we've actually modified the exception bitmap. */
9407	bool fModifiedXcptBitmap : 1;
9408
9409	/** We desire the modified the CR0 mask to be cleared. */
9410	bool fClearCr0Mask : 1;
9411	/** We desire the modified the CR4 mask to be cleared. */
9412	bool fClearCr4Mask : 1;
9413	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9414	uint32_t fCpe1Extra;
9415	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9416	uint32_t fCpe1Unwanted;
9417	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9418	uint32_t fCpe2Extra;
9419	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
9420	uint32_t bmXcptExtra;
9421	/** The sequence number of the Dtrace provider settings the state was
9422	* configured against. */
9423	uint32_t uDtraceSettingsSeqNo;
9424	/** VM-exits to check (one bit per VM-exit). */
9425	uint32_t bmExitsToCheck[3];
9426
9427	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9428	uint32_t fProcCtlsInitial;
9429	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9430	uint32_t fProcCtls2Initial;
9431	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9432	uint32_t bmXcptInitial;
9433	} VMXRUNDBGSTATE;
9434	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9435	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9436
9437
9438	/**
9439	* Initializes the VMXRUNDBGSTATE structure.
9440	*
9441	* @param pVCpu The cross context virtual CPU structure of the
9442	* calling EMT.
9443	* @param pCtx The CPU register context to go with @a pVCpu.
9444	* @param pDbgState The structure to initialize.
9445	*/
9446	DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9447	{
9448	pDbgState->uRipStart = pCtx->rip;
9449	pDbgState->uCsStart = pCtx->cs.Sel;
9450
9451	pDbgState->fModifiedProcCtls = false;
9452	pDbgState->fModifiedProcCtls2 = false;
9453	pDbgState->fModifiedXcptBitmap = false;
9454	pDbgState->fClearCr0Mask = false;
9455	pDbgState->fClearCr4Mask = false;
9456	pDbgState->fCpe1Extra = 0;
9457	pDbgState->fCpe1Unwanted = 0;
9458	pDbgState->fCpe2Extra = 0;
9459	pDbgState->bmXcptExtra = 0;
9460	pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9461	pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9462	pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9463	}
9464
9465
9466	/**
9467	* Updates the VMSC fields with changes requested by @a pDbgState.
9468	*
9469	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9470	* immediately before executing guest code, i.e. when interrupts are disabled.
9471	* We don't check status codes here as we cannot easily assert or return in the
9472	* latter case.
9473	*
9474	* @param pVCpu The cross context virtual CPU structure.
9475	* @param pDbgState The debug state.
9476	*/
9477	DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9478	{
9479	/*
9480	* Ensure desired flags in VMCS control fields are set.
9481	* (Ignoring write failure here, as we're committed and it's just debug extras.)
9482	*
9483	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9484	* there should be no stale data in pCtx at this point.
9485	*/
9486	if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9487	\|\| (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9488	{
9489	pVCpu->hm.s.vmx.u32ProcCtls \|= pDbgState->fCpe1Extra;
9490	pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9491	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9492	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9493	pDbgState->fModifiedProcCtls = true;
9494	}
9495
9496	if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9497	{
9498	pVCpu->hm.s.vmx.u32ProcCtls2 \|= pDbgState->fCpe2Extra;
9499	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9500	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9501	pDbgState->fModifiedProcCtls2 = true;
9502	}
9503
9504	if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9505	{
9506	pVCpu->hm.s.vmx.u32XcptBitmap \|= pDbgState->bmXcptExtra;
9507	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9508	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9509	pDbgState->fModifiedXcptBitmap = true;
9510	}
9511
9512	if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9513	{
9514	pVCpu->hm.s.vmx.u32CR0Mask = 0;
9515	VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9516	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9517	}
9518
9519	if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9520	{
9521	pVCpu->hm.s.vmx.u32CR4Mask = 0;
9522	VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9523	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9524	}
9525	}
9526
9527
9528	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9529	{
9530	/*
9531	* Restore VM-exit control settings as we may not reenter this function the
9532	* next time around.
9533	*/
9534	/* We reload the initial value, trigger what we can of recalculations the
9535	next time around. From the looks of things, that's all that's required atm. */
9536	if (pDbgState->fModifiedProcCtls)
9537	{
9538	if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9539	pDbgState->fProcCtlsInitial \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9540	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9541	AssertRCReturn(rc2, rc2);
9542	pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9543	}
9544
9545	/* We're currently the only ones messing with this one, so just restore the
9546	cached value and reload the field. */
9547	if ( pDbgState->fModifiedProcCtls2
9548	&& pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9549	{
9550	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9551	AssertRCReturn(rc2, rc2);
9552	pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9553	}
9554
9555	/* If we've modified the exception bitmap, we restore it and trigger
9556	reloading and partial recalculation the next time around. */
9557	if (pDbgState->fModifiedXcptBitmap)
9558	pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9559
9560	return rcStrict;
9561	}
9562
9563
9564	/**
9565	* Configures VM-exit controls for current DBGF and DTrace settings.
9566	*
9567	* This updates @a pDbgState and the VMCS execution control fields to reflect
9568	* the necessary VM-exits demanded by DBGF and DTrace.
9569	*
9570	* @param pVM The cross context VM structure.
9571	* @param pVCpu The cross context virtual CPU structure.
9572	* @param pCtx Pointer to the guest-CPU context.
9573	* @param pDbgState The debug state.
9574	* @param pVmxTransient Pointer to the VMX transient structure. May update
9575	* fUpdateTscOffsettingAndPreemptTimer.
9576	*/
9577	static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9578	PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9579	{
9580	/*
9581	* Take down the dtrace serial number so we can spot changes.
9582	*/
9583	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9584	ASMCompilerBarrier();
9585
9586	/*
9587	* We'll rebuild most of the middle block of data members (holding the
9588	* current settings) as we go along here, so start by clearing it all.
9589	*/
9590	pDbgState->bmXcptExtra = 0;
9591	pDbgState->fCpe1Extra = 0;
9592	pDbgState->fCpe1Unwanted = 0;
9593	pDbgState->fCpe2Extra = 0;
9594	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9595	pDbgState->bmExitsToCheck[i] = 0;
9596
9597	/*
9598	* Software interrupts (INT XXh) - no idea how to trigger these...
9599	*/
9600	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9601	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
9602	{
9603	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9604	}
9605
9606	/*
9607	* INT3 breakpoints - triggered by #BP exceptions.
9608	*/
9609	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9610	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
9611
9612	/*
9613	* Exception bitmap and XCPT events+probes.
9614	*/
9615	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9616	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9617	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
9618
9619	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
9620	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
9621	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
9622	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
9623	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
9624	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
9625	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
9626	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
9627	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
9628	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
9629	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
9630	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
9631	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
9632	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
9633	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
9634	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
9635	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
9636	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
9637
9638	if (pDbgState->bmXcptExtra)
9639	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9640
9641	/*
9642	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9643	*
9644	* Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9645	* So, when adding/changing/removing please don't forget to update it.
9646	*
9647	* Some of the macros are picking up local variables to save horizontal space,
9648	* (being able to see it in a table is the lesser evil here).
9649	*/
9650	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9651	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9652	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9653	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9654	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9655	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9656	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9657	} else do { } while (0)
9658	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9659	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9660	{ \
9661	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
9662	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9663	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9664	} else do { } while (0)
9665	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9666	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9667	{ \
9668	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
9669	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9670	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9671	} else do { } while (0)
9672	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9673	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9674	{ \
9675	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
9676	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9677	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9678	} else do { } while (0)
9679
9680	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9681	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9682	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9683	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9684	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9685
9686	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9687	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9688	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9689	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9690	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9691	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9692	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9693	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9694	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9695	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9696	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9697	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9698	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9699	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9700	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9701	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9702	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9703	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9704	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9705	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9706	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9707	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9708	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9709	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9710	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9711	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9712	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9713	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9714	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9715	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9716	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9717	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9718	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9719	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9720	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9721	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9722
9723	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9724	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9725	{
9726	int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9727	rc2 \|= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9728	rc2 \|= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9729	AssertRC(rc2);
9730
9731	#if 0 /** @todo fix me */
9732	pDbgState->fClearCr0Mask = true;
9733	pDbgState->fClearCr4Mask = true;
9734	#endif
9735	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9736	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9737	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9738	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9739	pDbgState->fCpe1Unwanted \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9740	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9741	require clearing here and in the loop if we start using it. */
9742	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9743	}
9744	else
9745	{
9746	if (pDbgState->fClearCr0Mask)
9747	{
9748	pDbgState->fClearCr0Mask = false;
9749	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9750	}
9751	if (pDbgState->fClearCr4Mask)
9752	{
9753	pDbgState->fClearCr4Mask = false;
9754	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9755	}
9756	}
9757	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9758	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9759
9760	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9761	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9762	{
9763	/** @todo later, need to fix handler as it assumes this won't usually happen. */
9764	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9765	}
9766	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9767	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9768
9769	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9770	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9771	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9772	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9773	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9774	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9775	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9776	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9777	#if 0 /** @todo too slow, fix handler. */
9778	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9779	#endif
9780	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9781
9782	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9783	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9784	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9785	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9786	{
9787	pDbgState->fCpe2Extra \|= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9788	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9789	}
9790	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9791	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9792	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9793	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9794
9795	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9796	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
9797	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9798	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
9799	{
9800	pDbgState->fCpe2Extra \|= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9801	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9802	}
9803	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9804	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9805	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9806	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9807
9808	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9809	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9810	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9811	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9812	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9813	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9814	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9815	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9816	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9817	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9818	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9819	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9820	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9821	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9822	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9823	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9824	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9825	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9826	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9827	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9828	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9829	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9830
9831	#undef IS_EITHER_ENABLED
9832	#undef SET_ONLY_XBM_IF_EITHER_EN
9833	#undef SET_CPE1_XBM_IF_EITHER_EN
9834	#undef SET_CPEU_XBM_IF_EITHER_EN
9835	#undef SET_CPE2_XBM_IF_EITHER_EN
9836
9837	/*
9838	* Sanitize the control stuff.
9839	*/
9840	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9841	if (pDbgState->fCpe2Extra)
9842	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9843	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9844	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9845	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9846	{
9847	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9848	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9849	}
9850
9851	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9852	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9853	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9854	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9855	}
9856
9857
9858	/**
9859	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
9860	* appropriate.
9861	*
9862	* The caller has checked the VM-exit against the
9863	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9864	* already, so we don't have to do that either.
9865	*
9866	* @returns Strict VBox status code (i.e. informational status codes too).
9867	* @param pVM The cross context VM structure.
9868	* @param pVCpu The cross context virtual CPU structure.
9869	* @param pMixedCtx Pointer to the guest-CPU context.
9870	* @param pVmxTransient Pointer to the VMX-transient structure.
9871	* @param uExitReason The VM-exit reason.
9872	*
9873	* @remarks The name of this function is displayed by dtrace, so keep it short
9874	* and to the point. No longer than 33 chars long, please.
9875	*/
9876	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9877	PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9878	{
9879	/*
9880	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9881	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
9882	*
9883	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9884	* does. Must add/change/remove both places. Same ordering, please.
9885	*
9886	* Added/removed events must also be reflected in the next section
9887	* where we dispatch dtrace events.
9888	*/
9889	bool fDtrace1 = false;
9890	bool fDtrace2 = false;
9891	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9892	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9893	uint32_t uEventArg = 0;
9894	#define SET_EXIT(a_EventSubName) \
9895	do { \
9896	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9897	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9898	} while (0)
9899	#define SET_BOTH(a_EventSubName) \
9900	do { \
9901	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9902	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9903	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9904	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9905	} while (0)
9906	switch (uExitReason)
9907	{
9908	case VMX_EXIT_MTF:
9909	return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9910
9911	case VMX_EXIT_XCPT_OR_NMI:
9912	{
9913	uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9914	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9915	{
9916	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9917	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9918	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9919	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9920	{
9921	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9922	{
9923	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9924	uEventArg = pVmxTransient->uExitIntErrorCode;
9925	}
9926	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9927	switch (enmEvent1)
9928	{
9929	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9930	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9931	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9932	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9933	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9934	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9935	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9936	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9937	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9938	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9939	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9940	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9941	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9942	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9943	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9944	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9945	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9946	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9947	default: break;
9948	}
9949	}
9950	else
9951	AssertFailed();
9952	break;
9953
9954	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9955	uEventArg = idxVector;
9956	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9957	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9958	break;
9959	}
9960	break;
9961	}
9962
9963	case VMX_EXIT_TRIPLE_FAULT:
9964	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9965	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9966	break;
9967	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9968	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9969	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9970	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9971	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9972
9973	/* Instruction specific VM-exits: */
9974	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9975	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9976	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9977	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9978	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9979	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9980	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9981	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9982	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9983	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9984	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9985	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9986	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9987	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9988	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9989	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9990	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9991	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9992	case VMX_EXIT_MOV_CRX:
9993	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9994	/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
9995	* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
9996	if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
9997	== VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
9998	SET_BOTH(CRX_READ);
9999	else
10000	SET_BOTH(CRX_WRITE);
10001	uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
10002	break;
10003	case VMX_EXIT_MOV_DRX:
10004	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10005	if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
10006	== VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
10007	SET_BOTH(DRX_READ);
10008	else
10009	SET_BOTH(DRX_WRITE);
10010	uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
10011	break;
10012	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
10013	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
10014	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
10015	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
10016	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
10017	case VMX_EXIT_XDTR_ACCESS:
10018	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10019	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
10020	{
10021	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
10022	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
10023	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
10024	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
10025	}
10026	break;
10027
10028	case VMX_EXIT_TR_ACCESS:
10029	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10030	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
10031	{
10032	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
10033	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
10034	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
10035	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
10036	}
10037	break;
10038
10039	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
10040	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
10041	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
10042	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
10043	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
10044	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
10045	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
10046	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
10047	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
10048	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
10049	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
10050
10051	/* Events that aren't relevant at this point. */
10052	case VMX_EXIT_EXT_INT:
10053	case VMX_EXIT_INT_WINDOW:
10054	case VMX_EXIT_NMI_WINDOW:
10055	case VMX_EXIT_TPR_BELOW_THRESHOLD:
10056	case VMX_EXIT_PREEMPT_TIMER:
10057	case VMX_EXIT_IO_INSTR:
10058	break;
10059
10060	/* Errors and unexpected events. */
10061	case VMX_EXIT_INIT_SIGNAL:
10062	case VMX_EXIT_SIPI:
10063	case VMX_EXIT_IO_SMI:
10064	case VMX_EXIT_SMI:
10065	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10066	case VMX_EXIT_ERR_MSR_LOAD:
10067	case VMX_EXIT_ERR_MACHINE_CHECK:
10068	break;
10069
10070	default:
10071	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10072	break;
10073	}
10074	#undef SET_BOTH
10075	#undef SET_EXIT
10076
10077	/*
10078	* Dtrace tracepoints go first. We do them here at once so we don't
10079	* have to copy the guest state saving and stuff a few dozen times.
10080	* Down side is that we've got to repeat the switch, though this time
10081	* we use enmEvent since the probes are a subset of what DBGF does.
10082	*/
10083	if (fDtrace1 \|\| fDtrace2)
10084	{
10085	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10086	hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10087	switch (enmEvent1)
10088	{
10089	/** @todo consider which extra parameters would be helpful for each probe. */
10090	case DBGFEVENT_END: break;
10091	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
10092	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
10093	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
10094	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
10095	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
10096	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
10097	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
10098	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
10099	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
10100	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
10101	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
10102	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
10103	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
10104	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
10105	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
10106	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
10107	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
10108	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
10109	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10110	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
10111	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
10112	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
10113	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
10114	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
10115	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
10116	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
10117	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
10118	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10119	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10120	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10121	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10122	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
10123	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
10124	RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
10125	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
10126	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
10127	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
10128	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
10129	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
10130	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
10131	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
10132	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
10133	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
10134	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
10135	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
10136	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
10137	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
10138	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
10139	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
10140	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
10141	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
10142	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
10143	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
10144	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
10145	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
10146	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
10147	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
10148	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
10149	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
10150	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
10151	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
10152	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
10153	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
10154	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
10155	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
10156	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
10157	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
10158	}
10159	switch (enmEvent2)
10160	{
10161	/** @todo consider which extra parameters would be helpful for each probe. */
10162	case DBGFEVENT_END: break;
10163	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
10164	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
10165	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
10166	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
10167	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
10168	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
10169	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
10170	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
10171	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
10172	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10173	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10174	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10175	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10176	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
10177	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
10178	RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
10179	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
10180	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
10181	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
10182	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
10183	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
10184	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
10185	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
10186	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
10187	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
10188	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
10189	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
10190	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
10191	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
10192	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
10193	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
10194	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
10195	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
10196	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
10197	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
10198	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
10199	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
10200	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
10201	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
10202	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
10203	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
10204	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
10205	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
10206	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
10207	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
10208	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
10209	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
10210	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
10211	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
10212	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
10213	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
10214	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
10215	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
10216	}
10217	}
10218
10219	/*
10220	* Fire of the DBGF event, if enabled (our check here is just a quick one,
10221	* the DBGF call will do a full check).
10222	*
10223	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
10224	* Note! If we have to events, we prioritize the first, i.e. the instruction
10225	* one, in order to avoid event nesting.
10226	*/
10227	if ( enmEvent1 != DBGFEVENT_END
10228	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
10229	{
10230	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
10231	if (rcStrict != VINF_SUCCESS)
10232	return rcStrict;
10233	}
10234	else if ( enmEvent2 != DBGFEVENT_END
10235	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
10236	{
10237	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
10238	if (rcStrict != VINF_SUCCESS)
10239	return rcStrict;
10240	}
10241
10242	return VINF_SUCCESS;
10243	}
10244
10245
10246	/**
10247	* Single-stepping VM-exit filtering.
10248	*
10249	* This is preprocessing the VM-exits and deciding whether we've gotten far
10250	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
10251	* handling is performed.
10252	*
10253	* @returns Strict VBox status code (i.e. informational status codes too).
10254	* @param pVM The cross context VM structure.
10255	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
10256	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
10257	* out-of-sync. Make sure to update the required
10258	* fields before using them.
10259	* @param pVmxTransient Pointer to the VMX-transient structure.
10260	* @param uExitReason The VM-exit reason.
10261	* @param pDbgState The debug state.
10262	*/
10263	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
10264	uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
10265	{
10266	/*
10267	* Expensive (saves context) generic dtrace VM-exit probe.
10268	*/
10269	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
10270	{ /* more likely */ }
10271	else
10272	{
10273	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10274	hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10275	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
10276	}
10277
10278	/*
10279	* Check for host NMI, just to get that out of the way.
10280	*/
10281	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
10282	{ /* normally likely */ }
10283	else
10284	{
10285	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10286	AssertRCReturn(rc2, rc2);
10287	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10288	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10289	return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
10290	}
10291
10292	/*
10293	* Check for single stepping event if we're stepping.
10294	*/
10295	if (pVCpu->hm.s.fSingleInstruction)
10296	{
10297	switch (uExitReason)
10298	{
10299	case VMX_EXIT_MTF:
10300	return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
10301
10302	/* Various events: */
10303	case VMX_EXIT_XCPT_OR_NMI:
10304	case VMX_EXIT_EXT_INT:
10305	case VMX_EXIT_TRIPLE_FAULT:
10306	case VMX_EXIT_INT_WINDOW:
10307	case VMX_EXIT_NMI_WINDOW:
10308	case VMX_EXIT_TASK_SWITCH:
10309	case VMX_EXIT_TPR_BELOW_THRESHOLD:
10310	case VMX_EXIT_APIC_ACCESS:
10311	case VMX_EXIT_EPT_VIOLATION:
10312	case VMX_EXIT_EPT_MISCONFIG:
10313	case VMX_EXIT_PREEMPT_TIMER:
10314
10315	/* Instruction specific VM-exits: */
10316	case VMX_EXIT_CPUID:
10317	case VMX_EXIT_GETSEC:
10318	case VMX_EXIT_HLT:
10319	case VMX_EXIT_INVD:
10320	case VMX_EXIT_INVLPG:
10321	case VMX_EXIT_RDPMC:
10322	case VMX_EXIT_RDTSC:
10323	case VMX_EXIT_RSM:
10324	case VMX_EXIT_VMCALL:
10325	case VMX_EXIT_VMCLEAR:
10326	case VMX_EXIT_VMLAUNCH:
10327	case VMX_EXIT_VMPTRLD:
10328	case VMX_EXIT_VMPTRST:
10329	case VMX_EXIT_VMREAD:
10330	case VMX_EXIT_VMRESUME:
10331	case VMX_EXIT_VMWRITE:
10332	case VMX_EXIT_VMXOFF:
10333	case VMX_EXIT_VMXON:
10334	case VMX_EXIT_MOV_CRX:
10335	case VMX_EXIT_MOV_DRX:
10336	case VMX_EXIT_IO_INSTR:
10337	case VMX_EXIT_RDMSR:
10338	case VMX_EXIT_WRMSR:
10339	case VMX_EXIT_MWAIT:
10340	case VMX_EXIT_MONITOR:
10341	case VMX_EXIT_PAUSE:
10342	case VMX_EXIT_XDTR_ACCESS:
10343	case VMX_EXIT_TR_ACCESS:
10344	case VMX_EXIT_INVEPT:
10345	case VMX_EXIT_RDTSCP:
10346	case VMX_EXIT_INVVPID:
10347	case VMX_EXIT_WBINVD:
10348	case VMX_EXIT_XSETBV:
10349	case VMX_EXIT_RDRAND:
10350	case VMX_EXIT_INVPCID:
10351	case VMX_EXIT_VMFUNC:
10352	case VMX_EXIT_RDSEED:
10353	case VMX_EXIT_XSAVES:
10354	case VMX_EXIT_XRSTORS:
10355	{
10356	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10357	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10358	AssertRCReturn(rc2, rc2);
10359	if ( pMixedCtx->rip != pDbgState->uRipStart
10360	\|\| pMixedCtx->cs.Sel != pDbgState->uCsStart)
10361	return VINF_EM_DBG_STEPPED;
10362	break;
10363	}
10364
10365	/* Errors and unexpected events: */
10366	case VMX_EXIT_INIT_SIGNAL:
10367	case VMX_EXIT_SIPI:
10368	case VMX_EXIT_IO_SMI:
10369	case VMX_EXIT_SMI:
10370	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10371	case VMX_EXIT_ERR_MSR_LOAD:
10372	case VMX_EXIT_ERR_MACHINE_CHECK:
10373	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
10374	break;
10375
10376	default:
10377	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10378	break;
10379	}
10380	}
10381
10382	/*
10383	* Check for debugger event breakpoints and dtrace probes.
10384	*/
10385	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
10386	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
10387	{
10388	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10389	if (rcStrict != VINF_SUCCESS)
10390	return rcStrict;
10391	}
10392
10393	/*
10394	* Normal processing.
10395	*/
10396	#ifdef HMVMX_USE_FUNCTION_TABLE
10397	return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
10398	#else
10399	return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10400	#endif
10401	}
10402
10403
10404	/**
10405	* Single steps guest code using VT-x.
10406	*
10407	* @returns Strict VBox status code (i.e. informational status codes too).
10408	* @param pVM The cross context VM structure.
10409	* @param pVCpu The cross context virtual CPU structure.
10410	* @param pCtx Pointer to the guest-CPU context.
10411	*
10412	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10413	*/
10414	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10415	{
10416	VMXTRANSIENT VmxTransient;
10417	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10418
10419	/* Set HMCPU indicators. */
10420	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10421	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
10422	pVCpu->hm.s.fDebugWantRdTscExit = false;
10423	pVCpu->hm.s.fUsingDebugLoop = true;
10424
10425	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10426	VMXRUNDBGSTATE DbgState;
10427	hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10428	hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10429
10430	/*
10431	* The loop.
10432	*/
10433	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10434	for (uint32_t cLoops = 0; ; cLoops++)
10435	{
10436	Assert(!HMR0SuspendPending());
10437	HMVMX_ASSERT_CPU_SAFE();
10438	bool fStepping = pVCpu->hm.s.fSingleInstruction;
10439
10440	/*
10441	* Preparatory work for running guest code, this may force us to return
10442	* to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10443	*/
10444	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10445	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10446	rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10447	if (rcStrict != VINF_SUCCESS)
10448	break;
10449
10450	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10451	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10452
10453	/*
10454	* Now we can run the guest code.
10455	*/
10456	int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10457
10458	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10459
10460	/*
10461	* Restore any residual host-state and save any bits shared between host
10462	* and guest into the guest-CPU state. Re-enables interrupts!
10463	*/
10464	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
10465
10466	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10467	if (RT_SUCCESS(rcRun))
10468	{ /* very likely */ }
10469	else
10470	{
10471	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10472	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10473	return rcRun;
10474	}
10475
10476	/* Profile the VM-exit. */
10477	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10478	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10479	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10480	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10481	HMVMX_START_EXIT_DISPATCH_PROF();
10482
10483	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10484
10485	/*
10486	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10487	*/
10488	rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10489	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10490	if (rcStrict != VINF_SUCCESS)
10491	break;
10492	if (cLoops > pVM->hm.s.cMaxResumeLoops)
10493	{
10494	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10495	rcStrict = VINF_EM_RAW_INTERRUPT;
10496	break;
10497	}
10498
10499	/*
10500	* Stepping: Did the RIP change, if so, consider it a single step.
10501	* Otherwise, make sure one of the TFs gets set.
10502	*/
10503	if (fStepping)
10504	{
10505	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10506	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10507	AssertRCReturn(rc2, rc2);
10508	if ( pCtx->rip != DbgState.uRipStart
10509	\|\| pCtx->cs.Sel != DbgState.uCsStart)
10510	{
10511	rcStrict = VINF_EM_DBG_STEPPED;
10512	break;
10513	}
10514	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10515	}
10516
10517	/*
10518	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
10519	*/
10520	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10521	hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10522	}
10523
10524	/*
10525	* Clear the X86_EFL_TF if necessary.
10526	*/
10527	if (pVCpu->hm.s.fClearTrapFlag)
10528	{
10529	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10530	AssertRCReturn(rc2, rc2);
10531	pVCpu->hm.s.fClearTrapFlag = false;
10532	pCtx->eflags.Bits.u1TF = 0;
10533	}
10534	/** @todo there seems to be issues with the resume flag when the monitor trap
10535	* flag is pending without being used. Seen early in bios init when
10536	* accessing APIC page in protected mode. */
10537
10538	/*
10539	* Restore VM-exit control settings as we may not reenter this function the
10540	* next time around.
10541	*/
10542	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10543
10544	/* Restore HMCPU indicators. */
10545	pVCpu->hm.s.fUsingDebugLoop = false;
10546	pVCpu->hm.s.fDebugWantRdTscExit = false;
10547	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10548
10549	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10550	return rcStrict;
10551	}
10552
10553
10554	/** @} */
10555
10556
10557	/**
10558	* Checks if any expensive dtrace probes are enabled and we should go to the
10559	* debug loop.
10560	*
10561	* @returns true if we should use debug loop, false if not.
10562	*/
10563	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10564	{
10565	/* It's probably faster to OR the raw 32-bit counter variables together.
10566	Since the variables are in an array and the probes are next to one
10567	another (more or less), we have good locality. So, better read
10568	eight-nine cache lines ever time and only have one conditional, than
10569	128+ conditionals, right? */
10570	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10571	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
10572	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
10573	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
10574	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
10575	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
10576	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
10577	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
10578	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
10579	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
10580	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
10581	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
10582	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
10583	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
10584	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
10585	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
10586	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
10587	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
10588	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
10589	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10590	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10591	) != 0
10592	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10593	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10594	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10595	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10596	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
10597	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10598	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10599	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10600	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10601	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10602	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10603	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10604	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10605	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10606	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10607	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10608	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10609	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10610	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10611	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10612	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10613	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10614	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10615	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10616	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
10617	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
10618	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10619	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
10620	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10621	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10622	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10623	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10624	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10625	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10626	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10627	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10628	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10629	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10630	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10631	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10632	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10633	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10634	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10635	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10636	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10637	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10638	) != 0
10639	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10640	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
10641	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10642	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10643	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10644	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
10645	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10646	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10647	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10648	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10649	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10650	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10651	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10652	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10653	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10654	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10655	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10656	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10657	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10658	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10659	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10660	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10661	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10662	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10663	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10664	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
10665	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
10666	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10667	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
10668	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10669	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10670	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10671	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10672	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10673	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10674	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10675	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10676	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10677	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10678	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10679	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10680	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10681	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10682	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10683	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10684	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10685	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10686	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10687	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10688	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10689	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10690	) != 0;
10691	}
10692
10693
10694	/**
10695	* Runs the guest code using VT-x.
10696	*
10697	* @returns Strict VBox status code (i.e. informational status codes too).
10698	* @param pVM The cross context VM structure.
10699	* @param pVCpu The cross context virtual CPU structure.
10700	* @param pCtx Pointer to the guest-CPU context.
10701	*/
10702	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10703	{
10704	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10705	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10706	HMVMX_ASSERT_PREEMPT_SAFE();
10707
10708	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10709
10710	VBOXSTRICTRC rcStrict;
10711	if ( !pVCpu->hm.s.fUseDebugLoop
10712	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
10713	&& !DBGFIsStepping(pVCpu)
10714	&& !pVM->dbgf.ro.cEnabledInt3Breakpoints)
10715	rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10716	else
10717	rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10718
10719	if (rcStrict == VERR_EM_INTERPRETER)
10720	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10721	else if (rcStrict == VINF_EM_RESET)
10722	rcStrict = VINF_EM_TRIPLE_FAULT;
10723
10724	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10725	if (RT_FAILURE(rc2))
10726	{
10727	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10728	rcStrict = rc2;
10729	}
10730	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10731	return rcStrict;
10732	}
10733
10734
10735	#ifndef HMVMX_USE_FUNCTION_TABLE
10736	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10737	{
10738	# ifdef DEBUG_ramshankar
10739	# define RETURN_EXIT_CALL(a_CallExpr) \
10740	do { \
10741	int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10742	VBOXSTRICTRC rcStrict = a_CallExpr; \
10743	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10744	return rcStrict; \
10745	} while (0)
10746	# else
10747	# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10748	# endif
10749	switch (rcReason)
10750	{
10751	case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10752	case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10753	case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10754	case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10755	case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10756	case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10757	case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10758	case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10759	case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10760	case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10761	case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10762	case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10763	case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10764	case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10765	case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10766	case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10767	case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10768	case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10769	case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10770	case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10771	case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10772	case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10773	case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10774	case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10775	case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10776	case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10777	case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10778	case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10779	case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10780	case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10781	case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10782	case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10783	case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10784	case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10785
10786	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10787	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10788	case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10789	case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10790	case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10791	case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10792	case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10793	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10794	case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10795
10796	case VMX_EXIT_VMCLEAR:
10797	case VMX_EXIT_VMLAUNCH:
10798	case VMX_EXIT_VMPTRLD:
10799	case VMX_EXIT_VMPTRST:
10800	case VMX_EXIT_VMREAD:
10801	case VMX_EXIT_VMRESUME:
10802	case VMX_EXIT_VMWRITE:
10803	case VMX_EXIT_VMXOFF:
10804	case VMX_EXIT_VMXON:
10805	case VMX_EXIT_INVEPT:
10806	case VMX_EXIT_INVVPID:
10807	case VMX_EXIT_VMFUNC:
10808	case VMX_EXIT_XSAVES:
10809	case VMX_EXIT_XRSTORS:
10810	return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10811	case VMX_EXIT_ENCLS:
10812	case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10813	case VMX_EXIT_PML_FULL:
10814	default:
10815	return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10816	}
10817	#undef RETURN_EXIT_CALL
10818	}
10819	#endif /* !HMVMX_USE_FUNCTION_TABLE */
10820
10821
10822	#ifdef VBOX_STRICT
10823	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10824	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10825	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10826
10827	# define HMVMX_ASSERT_PREEMPT_CPUID() \
10828	do { \
10829	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10830	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10831	} while (0)
10832
10833	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10834	do { \
10835	AssertPtr(pVCpu); \
10836	AssertPtr(pMixedCtx); \
10837	AssertPtr(pVmxTransient); \
10838	Assert(pVmxTransient->fVMEntryFailed == false); \
10839	Assert(ASMIntAreEnabled()); \
10840	HMVMX_ASSERT_PREEMPT_SAFE(); \
10841	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10842	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10843	HMVMX_ASSERT_PREEMPT_SAFE(); \
10844	if (VMMR0IsLogFlushDisabled(pVCpu)) \
10845	HMVMX_ASSERT_PREEMPT_CPUID(); \
10846	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10847	} while (0)
10848
10849	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10850	do { \
10851	Log4Func(("\n")); \
10852	} while (0)
10853	#else /* nonstrict builds: */
10854	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10855	do { \
10856	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10857	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10858	} while (0)
10859	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10860	#endif
10861
10862
10863	/**
10864	* Advances the guest RIP by the specified number of bytes.
10865	*
10866	* @param pVCpu The cross context virtual CPU structure.
10867	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10868	* out-of-sync. Make sure to update the required fields
10869	* before using them.
10870	* @param cbInstr Number of bytes to advance the RIP by.
10871	*
10872	* @remarks No-long-jump zone!!!
10873	*/
10874	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10875	{
10876	/* Advance the RIP. */
10877	pMixedCtx->rip += cbInstr;
10878	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10879
10880	/* Update interrupt inhibition. */
10881	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10882	&& pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10883	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10884	}
10885
10886
10887	/**
10888	* Advances the guest RIP after reading it from the VMCS.
10889	*
10890	* @returns VBox status code, no informational status codes.
10891	* @param pVCpu The cross context virtual CPU structure.
10892	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10893	* out-of-sync. Make sure to update the required fields
10894	* before using them.
10895	* @param pVmxTransient Pointer to the VMX transient structure.
10896	*
10897	* @remarks No-long-jump zone!!!
10898	*/
10899	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10900	{
10901	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10902	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10903	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10904	AssertRCReturn(rc, rc);
10905
10906	hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10907
10908	/*
10909	* Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10910	* pending debug exception field as it takes care of priority of events.
10911	*
10912	* See Intel spec. 32.2.1 "Debug Exceptions".
10913	*/
10914	if ( !pVCpu->hm.s.fSingleInstruction
10915	&& pMixedCtx->eflags.Bits.u1TF)
10916	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10917
10918	return VINF_SUCCESS;
10919	}
10920
10921
10922	/**
10923	* Tries to determine what part of the guest-state VT-x has deemed as invalid
10924	* and update error record fields accordingly.
10925	*
10926	* @return VMX_IGS_* return codes.
10927	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10928	* wrong with the guest state.
10929	*
10930	* @param pVM The cross context VM structure.
10931	* @param pVCpu The cross context virtual CPU structure.
10932	* @param pCtx Pointer to the guest-CPU state.
10933	*
10934	* @remarks This function assumes our cache of the VMCS controls
10935	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10936	*/
10937	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10938	{
10939	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10940	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10941	uError = (err); \
10942	break; \
10943	} else do { } while (0)
10944
10945	int rc;
10946	uint32_t uError = VMX_IGS_ERROR;
10947	uint32_t u32Val;
10948	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10949
10950	do
10951	{
10952	/*
10953	* CR0.
10954	*/
10955	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10956	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10957	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10958	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10959	if (fUnrestrictedGuest)
10960	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
10961
10962	uint32_t u32GuestCR0;
10963	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
10964	AssertRCBreak(rc);
10965	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
10966	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
10967	if ( !fUnrestrictedGuest
10968	&& (u32GuestCR0 & X86_CR0_PG)
10969	&& !(u32GuestCR0 & X86_CR0_PE))
10970	{
10971	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10972	}
10973
10974	/*
10975	* CR4.
10976	*/
10977	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10978	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10979
10980	uint32_t u32GuestCR4;
10981	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
10982	AssertRCBreak(rc);
10983	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
10984	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
10985
10986	/*
10987	* IA32_DEBUGCTL MSR.
10988	*/
10989	uint64_t u64Val;
10990	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10991	AssertRCBreak(rc);
10992	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10993	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10994	{
10995	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10996	}
10997	uint64_t u64DebugCtlMsr = u64Val;
10998
10999	#ifdef VBOX_STRICT
11000	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
11001	AssertRCBreak(rc);
11002	Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
11003	#endif
11004	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
11005
11006	/*
11007	* RIP and RFLAGS.
11008	*/
11009	uint32_t u32Eflags;
11010	#if HC_ARCH_BITS == 64
11011	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
11012	AssertRCBreak(rc);
11013	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
11014	if ( !fLongModeGuest
11015	\|\| !pCtx->cs.Attr.n.u1Long)
11016	{
11017	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
11018	}
11019	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
11020	* must be identical if the "IA-32e mode guest" VM-entry
11021	* control is 1 and CS.L is 1. No check applies if the
11022	* CPU supports 64 linear-address bits. */
11023
11024	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
11025	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
11026	AssertRCBreak(rc);
11027	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
11028	VMX_IGS_RFLAGS_RESERVED);
11029	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
11030	u32Eflags = u64Val;
11031	#else
11032	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
11033	AssertRCBreak(rc);
11034	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
11035	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
11036	#endif
11037
11038	if ( fLongModeGuest
11039	\|\| ( fUnrestrictedGuest
11040	&& !(u32GuestCR0 & X86_CR0_PE)))
11041	{
11042	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
11043	}
11044
11045	uint32_t u32EntryInfo;
11046	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
11047	AssertRCBreak(rc);
11048	if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11049	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11050	{
11051	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
11052	}
11053
11054	/*
11055	* 64-bit checks.
11056	*/
11057	#if HC_ARCH_BITS == 64
11058	if (fLongModeGuest)
11059	{
11060	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
11061	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
11062	}
11063
11064	if ( !fLongModeGuest
11065	&& (u32GuestCR4 & X86_CR4_PCIDE))
11066	{
11067	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
11068	}
11069
11070	/** @todo CR3 field must be such that bits 63:52 and bits in the range
11071	* 51:32 beyond the processor's physical-address width are 0. */
11072
11073	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
11074	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
11075	{
11076	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
11077	}
11078
11079	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
11080	AssertRCBreak(rc);
11081	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
11082
11083	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
11084	AssertRCBreak(rc);
11085	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
11086	#endif
11087
11088	/*
11089	* PERF_GLOBAL MSR.
11090	*/
11091	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
11092	{
11093	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
11094	AssertRCBreak(rc);
11095	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
11096	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
11097	}
11098
11099	/*
11100	* PAT MSR.
11101	*/
11102	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
11103	{
11104	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
11105	AssertRCBreak(rc);
11106	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
11107	for (unsigned i = 0; i < 8; i++)
11108	{
11109	uint8_t u8Val = (u64Val & 0xff);
11110	if ( u8Val != 0 /* UC */
11111	&& u8Val != 1 /* WC */
11112	&& u8Val != 4 /* WT */
11113	&& u8Val != 5 /* WP */
11114	&& u8Val != 6 /* WB */
11115	&& u8Val != 7 /* UC- */)
11116	{
11117	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
11118	}
11119	u64Val >>= 8;
11120	}
11121	}
11122
11123	/*
11124	* EFER MSR.
11125	*/
11126	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
11127	{
11128	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
11129	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
11130	AssertRCBreak(rc);
11131	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
11132	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
11133	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
11134	& VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
11135	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
11136	HMVMX_CHECK_BREAK( fUnrestrictedGuest
11137	\|\| !(u32GuestCR0 & X86_CR0_PG)
11138	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
11139	VMX_IGS_EFER_LMA_LME_MISMATCH);
11140	}
11141
11142	/*
11143	* Segment registers.
11144	*/
11145	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11146	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
11147	if (!(u32Eflags & X86_EFL_VM))
11148	{
11149	/* CS */
11150	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
11151	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
11152	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
11153	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
11154	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
11155	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
11156	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
11157	/* CS cannot be loaded with NULL in protected mode. */
11158	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
11159	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
11160	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
11161	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
11162	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
11163	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
11164	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
11165	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
11166	else
11167	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
11168
11169	/* SS */
11170	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11171	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
11172	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
11173	if ( !(pCtx->cr0 & X86_CR0_PE)
11174	\|\| pCtx->cs.Attr.n.u4Type == 3)
11175	{
11176	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
11177	}
11178	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
11179	{
11180	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
11181	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
11182	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
11183	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
11184	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
11185	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
11186	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
11187	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
11188	}
11189
11190	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
11191	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
11192	{
11193	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
11194	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
11195	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11196	\|\| pCtx->ds.Attr.n.u4Type > 11
11197	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
11198	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
11199	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
11200	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
11201	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
11202	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
11203	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
11204	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11205	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
11206	}
11207	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
11208	{
11209	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
11210	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
11211	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11212	\|\| pCtx->es.Attr.n.u4Type > 11
11213	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
11214	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
11215	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
11216	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
11217	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
11218	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
11219	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
11220	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11221	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
11222	}
11223	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
11224	{
11225	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
11226	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
11227	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11228	\|\| pCtx->fs.Attr.n.u4Type > 11
11229	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
11230	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
11231	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
11232	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
11233	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
11234	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
11235	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
11236	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11237	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
11238	}
11239	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
11240	{
11241	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
11242	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
11243	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11244	\|\| pCtx->gs.Attr.n.u4Type > 11
11245	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
11246	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
11247	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
11248	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
11249	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
11250	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
11251	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
11252	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11253	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
11254	}
11255	/* 64-bit capable CPUs. */
11256	#if HC_ARCH_BITS == 64
11257	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11258	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11259	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11260	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11261	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11262	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
11263	VMX_IGS_LONGMODE_SS_BASE_INVALID);
11264	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
11265	VMX_IGS_LONGMODE_DS_BASE_INVALID);
11266	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
11267	VMX_IGS_LONGMODE_ES_BASE_INVALID);
11268	#endif
11269	}
11270	else
11271	{
11272	/* V86 mode checks. */
11273	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
11274	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11275	{
11276	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
11277	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
11278	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
11279	}
11280	else
11281	{
11282	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
11283	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
11284	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
11285	}
11286
11287	/* CS */
11288	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
11289	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
11290	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
11291	/* SS */
11292	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
11293	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
11294	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
11295	/* DS */
11296	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
11297	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
11298	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
11299	/* ES */
11300	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
11301	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
11302	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
11303	/* FS */
11304	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
11305	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
11306	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
11307	/* GS */
11308	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
11309	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
11310	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
11311	/* 64-bit capable CPUs. */
11312	#if HC_ARCH_BITS == 64
11313	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11314	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11315	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11316	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11317	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11318	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
11319	VMX_IGS_LONGMODE_SS_BASE_INVALID);
11320	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
11321	VMX_IGS_LONGMODE_DS_BASE_INVALID);
11322	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
11323	VMX_IGS_LONGMODE_ES_BASE_INVALID);
11324	#endif
11325	}
11326
11327	/*
11328	* TR.
11329	*/
11330	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
11331	/* 64-bit capable CPUs. */
11332	#if HC_ARCH_BITS == 64
11333	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
11334	#endif
11335	if (fLongModeGuest)
11336	{
11337	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
11338	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
11339	}
11340	else
11341	{
11342	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
11343	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
11344	VMX_IGS_TR_ATTR_TYPE_INVALID);
11345	}
11346	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
11347	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
11348	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
11349	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
11350	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11351	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
11352	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11353	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
11354
11355	/*
11356	* GDTR and IDTR.
11357	*/
11358	#if HC_ARCH_BITS == 64
11359	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
11360	AssertRCBreak(rc);
11361	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
11362
11363	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
11364	AssertRCBreak(rc);
11365	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
11366	#endif
11367
11368	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
11369	AssertRCBreak(rc);
11370	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11371
11372	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
11373	AssertRCBreak(rc);
11374	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11375
11376	/*
11377	* Guest Non-Register State.
11378	*/
11379	/* Activity State. */
11380	uint32_t u32ActivityState;
11381	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
11382	AssertRCBreak(rc);
11383	HMVMX_CHECK_BREAK( !u32ActivityState
11384	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
11385	VMX_IGS_ACTIVITY_STATE_INVALID);
11386	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
11387	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
11388	uint32_t u32IntrState;
11389	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
11390	AssertRCBreak(rc);
11391	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
11392	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11393	{
11394	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
11395	}
11396
11397	/** @todo Activity state and injecting interrupts. Left as a todo since we
11398	* currently don't use activity states but ACTIVE. */
11399
11400	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11401	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11402
11403	/* Guest interruptibility-state. */
11404	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11405	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11406	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
11407	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11408	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11409	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11410	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11411	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11412	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11413	if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
11414	{
11415	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11416	{
11417	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11418	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11419	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11420	}
11421	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11422	{
11423	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11424	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11425	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11426	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11427	}
11428	}
11429	/** @todo Assumes the processor is not in SMM. */
11430	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11431	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11432	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11433	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11434	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11435	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11436	&& VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11437	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11438	{
11439	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11440	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11441	}
11442
11443	/* Pending debug exceptions. */
11444	#if HC_ARCH_BITS == 64
11445	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11446	AssertRCBreak(rc);
11447	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11448	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11449	u32Val = u64Val; /* For pending debug exceptions checks below. */
11450	#else
11451	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11452	AssertRCBreak(rc);
11453	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11454	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11455	#endif
11456
11457	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11458	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11459	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11460	{
11461	if ( (u32Eflags & X86_EFL_TF)
11462	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11463	{
11464	/* Bit 14 is PendingDebug.BS. */
11465	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11466	}
11467	if ( !(u32Eflags & X86_EFL_TF)
11468	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11469	{
11470	/* Bit 14 is PendingDebug.BS. */
11471	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11472	}
11473	}
11474
11475	/* VMCS link pointer. */
11476	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11477	AssertRCBreak(rc);
11478	if (u64Val != UINT64_C(0xffffffffffffffff))
11479	{
11480	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11481	/** @todo Bits beyond the processor's physical-address width MBZ. */
11482	/** @todo 32-bit located in memory referenced by value of this field (as a
11483	* physical address) must contain the processor's VMCS revision ID. */
11484	/** @todo SMM checks. */
11485	}
11486
11487	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11488	* not using Nested Paging? */
11489	if ( pVM->hm.s.fNestedPaging
11490	&& !fLongModeGuest
11491	&& CPUMIsGuestInPAEModeEx(pCtx))
11492	{
11493	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11494	AssertRCBreak(rc);
11495	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11496
11497	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11498	AssertRCBreak(rc);
11499	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11500
11501	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11502	AssertRCBreak(rc);
11503	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11504
11505	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11506	AssertRCBreak(rc);
11507	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11508	}
11509
11510	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11511	if (uError == VMX_IGS_ERROR)
11512	uError = VMX_IGS_REASON_NOT_FOUND;
11513	} while (0);
11514
11515	pVCpu->hm.s.u32HMError = uError;
11516	return uError;
11517
11518	#undef HMVMX_ERROR_BREAK
11519	#undef HMVMX_CHECK_BREAK
11520	}
11521
11522	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11523	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11524	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11525
11526	/** @name VM-exit handlers.
11527	* @{
11528	*/
11529
11530	/**
11531	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11532	*/
11533	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11534	{
11535	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11536	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11537	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11538	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11539	return VINF_SUCCESS;
11540	return VINF_EM_RAW_INTERRUPT;
11541	}
11542
11543
11544	/**
11545	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11546	*/
11547	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11548	{
11549	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11550	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11551
11552	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11553	AssertRCReturn(rc, rc);
11554
11555	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11556	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11557	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11558	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11559
11560	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11561	{
11562	/*
11563	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11564	* anything we inject is not going to cause a VM-exit directly for the event being injected.
11565	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11566	*
11567	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11568	*/
11569	VMXDispatchHostNmi();
11570	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11571	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11572	return VINF_SUCCESS;
11573	}
11574
11575	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11576	VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11577	if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11578	{ /* likely */ }
11579	else
11580	{
11581	if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11582	rcStrictRc1 = VINF_SUCCESS;
11583	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11584	return rcStrictRc1;
11585	}
11586
11587	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11588	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11589	switch (uIntType)
11590	{
11591	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11592	Assert(uVector == X86_XCPT_DB);
11593	RT_FALL_THRU();
11594	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11595	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11596	RT_FALL_THRU();
11597	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11598	{
11599	/*
11600	* If there's any exception caused as a result of event injection, the resulting
11601	* secondary/final execption will be pending, we shall continue guest execution
11602	* after injecting the event. The page-fault case is complicated and we manually
11603	* handle any currently pending event in hmR0VmxExitXcptPF.
11604	*/
11605	if (!pVCpu->hm.s.Event.fPending)
11606	{ /* likely */ }
11607	else if (uVector != X86_XCPT_PF)
11608	{
11609	rc = VINF_SUCCESS;
11610	break;
11611	}
11612
11613	switch (uVector)
11614	{
11615	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11616	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11617	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
11618	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11619	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11620	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11621	case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11622
11623	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11624	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11625	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11626	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11627	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11628	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11629	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11630	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11631	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11632	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11633	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11634	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11635	default:
11636	{
11637	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11638	AssertRCReturn(rc, rc);
11639
11640	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11641	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11642	{
11643	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11644	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11645	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11646
11647	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11648	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11649	AssertRCReturn(rc, rc);
11650	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11651	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11652	0 /* GCPtrFaultAddress */);
11653	AssertRCReturn(rc, rc);
11654	}
11655	else
11656	{
11657	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11658	pVCpu->hm.s.u32HMError = uVector;
11659	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11660	}
11661	break;
11662	}
11663	}
11664	break;
11665	}
11666
11667	default:
11668	{
11669	pVCpu->hm.s.u32HMError = uExitIntInfo;
11670	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11671	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11672	break;
11673	}
11674	}
11675	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11676	return rc;
11677	}
11678
11679
11680	/**
11681	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11682	*/
11683	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11684	{
11685	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11686
11687	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11688	hmR0VmxClearIntWindowExitVmcs(pVCpu);
11689
11690	/* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11691	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11692	return VINF_SUCCESS;
11693	}
11694
11695
11696	/**
11697	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11698	*/
11699	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11700	{
11701	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11702	if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11703	{
11704	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11705	HMVMX_RETURN_UNEXPECTED_EXIT();
11706	}
11707
11708	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11709
11710	/*
11711	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11712	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11713	*/
11714	uint32_t uIntrState = 0;
11715	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11716	AssertRCReturn(rc, rc);
11717
11718	bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11719	if ( fBlockSti
11720	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11721	{
11722	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11723	}
11724
11725	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11726	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11727
11728	/* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11729	return VINF_SUCCESS;
11730	}
11731
11732
11733	/**
11734	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11735	*/
11736	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11737	{
11738	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11739	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11740	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11741	}
11742
11743
11744	/**
11745	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11746	*/
11747	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11748	{
11749	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11750	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11751	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11752	}
11753
11754
11755	/**
11756	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11757	*/
11758	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11759	{
11760	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11761	PVM pVM = pVCpu->CTX_SUFF(pVM);
11762	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11763	if (RT_LIKELY(rc == VINF_SUCCESS))
11764	{
11765	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11766	Assert(pVmxTransient->cbInstr == 2);
11767	}
11768	else
11769	{
11770	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11771	rc = VERR_EM_INTERPRETER;
11772	}
11773	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11774	return rc;
11775	}
11776
11777
11778	/**
11779	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11780	*/
11781	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11782	{
11783	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11784	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11785	AssertRCReturn(rc, rc);
11786
11787	if (pMixedCtx->cr4 & X86_CR4_SMXE)
11788	return VINF_EM_RAW_EMULATE_INSTR;
11789
11790	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11791	HMVMX_RETURN_UNEXPECTED_EXIT();
11792	}
11793
11794
11795	/**
11796	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11797	*/
11798	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11799	{
11800	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11801	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11802	AssertRCReturn(rc, rc);
11803
11804	PVM pVM = pVCpu->CTX_SUFF(pVM);
11805	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11806	if (RT_LIKELY(rc == VINF_SUCCESS))
11807	{
11808	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11809	Assert(pVmxTransient->cbInstr == 2);
11810	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11811	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11812	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11813	}
11814	else
11815	rc = VERR_EM_INTERPRETER;
11816	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11817	return rc;
11818	}
11819
11820
11821	/**
11822	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11823	*/
11824	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11825	{
11826	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11827	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11828	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11829	AssertRCReturn(rc, rc);
11830
11831	PVM pVM = pVCpu->CTX_SUFF(pVM);
11832	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11833	if (RT_SUCCESS(rc))
11834	{
11835	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11836	Assert(pVmxTransient->cbInstr == 3);
11837	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11838	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11839	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11840	}
11841	else
11842	{
11843	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11844	rc = VERR_EM_INTERPRETER;
11845	}
11846	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11847	return rc;
11848	}
11849
11850
11851	/**
11852	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11853	*/
11854	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11855	{
11856	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11857	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11858	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11859	AssertRCReturn(rc, rc);
11860
11861	PVM pVM = pVCpu->CTX_SUFF(pVM);
11862	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11863	if (RT_LIKELY(rc == VINF_SUCCESS))
11864	{
11865	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11866	Assert(pVmxTransient->cbInstr == 2);
11867	}
11868	else
11869	{
11870	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11871	rc = VERR_EM_INTERPRETER;
11872	}
11873	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11874	return rc;
11875	}
11876
11877
11878	/**
11879	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11880	*/
11881	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11882	{
11883	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11884	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11885
11886	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11887	if (pVCpu->hm.s.fHypercallsEnabled)
11888	{
11889	#if 0
11890	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11891	#else
11892	/* Aggressive state sync. for now. */
11893	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11894	rc \|= hmR0VmxSaveGuestRflags(pVCpu,pMixedCtx); /* For CPL checks in gimHvHypercall() & gimKvmHypercall() */
11895	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11896	AssertRCReturn(rc, rc);
11897	#endif
11898
11899	/* Perform the hypercall. */
11900	rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11901	if (rcStrict == VINF_SUCCESS)
11902	{
11903	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11904	AssertRCReturn(rc, rc);
11905	}
11906	else
11907	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11908	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11909	\|\| RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
11910
11911	/* If the hypercall changes anything other than guest's general-purpose registers,
11912	we would need to reload the guest changed bits here before VM-entry. */
11913	}
11914	else
11915	Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11916
11917	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11918	if (RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)))
11919	{
11920	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11921	rcStrict = VINF_SUCCESS;
11922	}
11923
11924	return rcStrict;
11925	}
11926
11927
11928	/**
11929	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11930	*/
11931	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11932	{
11933	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11934	PVM pVM = pVCpu->CTX_SUFF(pVM);
11935	Assert(!pVM->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
11936
11937	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11938	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11939	AssertRCReturn(rc, rc);
11940
11941	VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11942	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11943	rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11944	else
11945	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11946	pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11947	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
11948	return rcStrict;
11949	}
11950
11951
11952	/**
11953	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11954	*/
11955	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11956	{
11957	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11958	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11959	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11960	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11961	AssertRCReturn(rc, rc);
11962
11963	PVM pVM = pVCpu->CTX_SUFF(pVM);
11964	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11965	if (RT_LIKELY(rc == VINF_SUCCESS))
11966	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11967	else
11968	{
11969	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11970	rc = VERR_EM_INTERPRETER;
11971	}
11972	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11973	return rc;
11974	}
11975
11976
11977	/**
11978	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11979	*/
11980	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11981	{
11982	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11983	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11984	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11985	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11986	AssertRCReturn(rc, rc);
11987
11988	PVM pVM = pVCpu->CTX_SUFF(pVM);
11989	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11990	rc = VBOXSTRICTRC_VAL(rc2);
11991	if (RT_LIKELY( rc == VINF_SUCCESS
11992	\|\| rc == VINF_EM_HALT))
11993	{
11994	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11995	AssertRCReturn(rc3, rc3);
11996
11997	if ( rc == VINF_EM_HALT
11998	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11999	{
12000	rc = VINF_SUCCESS;
12001	}
12002	}
12003	else
12004	{
12005	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
12006	rc = VERR_EM_INTERPRETER;
12007	}
12008	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
12009	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
12010	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
12011	return rc;
12012	}
12013
12014
12015	/**
12016	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
12017	*/
12018	HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12019	{
12020	/*
12021	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
12022	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
12023	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
12024	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
12025	*/
12026	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12027	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12028	HMVMX_RETURN_UNEXPECTED_EXIT();
12029	}
12030
12031
12032	/**
12033	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
12034	*/
12035	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12036	{
12037	/*
12038	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
12039	* root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
12040	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
12041	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
12042	*/
12043	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12044	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12045	HMVMX_RETURN_UNEXPECTED_EXIT();
12046	}
12047
12048
12049	/**
12050	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
12051	*/
12052	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12053	{
12054	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
12055	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12056	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12057	HMVMX_RETURN_UNEXPECTED_EXIT();
12058	}
12059
12060
12061	/**
12062	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
12063	*/
12064	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12065	{
12066	/*
12067	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
12068	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
12069	* See Intel spec. 25.3 "Other Causes of VM-exits".
12070	*/
12071	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12072	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12073	HMVMX_RETURN_UNEXPECTED_EXIT();
12074	}
12075
12076
12077	/**
12078	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
12079	* VM-exit.
12080	*/
12081	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12082	{
12083	/*
12084	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
12085	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
12086	*
12087	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
12088	* See Intel spec. "23.8 Restrictions on VMX operation".
12089	*/
12090	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12091	return VINF_SUCCESS;
12092	}
12093
12094
12095	/**
12096	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
12097	* VM-exit.
12098	*/
12099	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12100	{
12101	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12102	return VINF_EM_RESET;
12103	}
12104
12105
12106	/**
12107	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
12108	*/
12109	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12110	{
12111	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12112	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
12113
12114	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12115	AssertRCReturn(rc, rc);
12116
12117	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
12118	rc = VINF_SUCCESS;
12119	else
12120	rc = VINF_EM_HALT;
12121
12122	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
12123	if (rc != VINF_SUCCESS)
12124	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
12125	return rc;
12126	}
12127
12128
12129	/**
12130	* VM-exit handler for instructions that result in a \#UD exception delivered to
12131	* the guest.
12132	*/
12133	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12134	{
12135	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12136	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
12137	return VINF_SUCCESS;
12138	}
12139
12140
12141	/**
12142	* VM-exit handler for expiry of the VMX preemption timer.
12143	*/
12144	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12145	{
12146	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12147
12148	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
12149	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12150
12151	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
12152	PVM pVM = pVCpu->CTX_SUFF(pVM);
12153	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
12154	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
12155	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
12156	}
12157
12158
12159	/**
12160	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
12161	*/
12162	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12163	{
12164	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12165
12166	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12167	rc \|= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /fMemory/, false /fNeedRsp/);
12168	rc \|= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
12169	AssertRCReturn(rc, rc);
12170
12171	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
12172	HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12173
12174	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
12175
12176	return rcStrict;
12177	}
12178
12179
12180	/**
12181	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
12182	*/
12183	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12184	{
12185	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12186
12187	/* The guest should not invalidate the host CPU's TLBs, fallback to interpreter. */
12188	/** @todo implement EMInterpretInvpcid() */
12189	return VERR_EM_INTERPRETER;
12190	}
12191
12192
12193	/**
12194	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
12195	* Error VM-exit.
12196	*/
12197	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12198	{
12199	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12200	AssertRCReturn(rc, rc);
12201
12202	rc = hmR0VmxCheckVmcsCtls(pVCpu);
12203	AssertRCReturn(rc, rc);
12204
12205	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12206	NOREF(uInvalidReason);
12207
12208	#ifdef VBOX_STRICT
12209	uint32_t uIntrState;
12210	RTHCUINTREG uHCReg;
12211	uint64_t u64Val;
12212	uint32_t u32Val;
12213
12214	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
12215	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
12216	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
12217	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
12218	AssertRCReturn(rc, rc);
12219
12220	Log4(("uInvalidReason %u\n", uInvalidReason));
12221	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
12222	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
12223	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
12224	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
12225
12226	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
12227	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
12228	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
12229	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
12230	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
12231	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
12232	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
12233	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
12234	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
12235	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
12236	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
12237	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
12238	#else
12239	NOREF(pVmxTransient);
12240	#endif
12241
12242	hmR0DumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12243	return VERR_VMX_INVALID_GUEST_STATE;
12244	}
12245
12246
12247	/**
12248	* VM-exit handler for VM-entry failure due to an MSR-load
12249	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
12250	*/
12251	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12252	{
12253	NOREF(pVmxTransient);
12254	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
12255	HMVMX_RETURN_UNEXPECTED_EXIT();
12256	}
12257
12258
12259	/**
12260	* VM-exit handler for VM-entry failure due to a machine-check event
12261	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
12262	*/
12263	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12264	{
12265	NOREF(pVmxTransient);
12266	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
12267	HMVMX_RETURN_UNEXPECTED_EXIT();
12268	}
12269
12270
12271	/**
12272	* VM-exit handler for all undefined reasons. Should never ever happen.. in
12273	* theory.
12274	*/
12275	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12276	{
12277	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
12278	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
12279	return VERR_VMX_UNDEFINED_EXIT_CODE;
12280	}
12281
12282
12283	/**
12284	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
12285	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
12286	* Conditional VM-exit.
12287	*/
12288	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12289	{
12290	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12291
12292	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
12293	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
12294	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
12295	return VERR_EM_INTERPRETER;
12296	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12297	HMVMX_RETURN_UNEXPECTED_EXIT();
12298	}
12299
12300
12301	/**
12302	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
12303	*/
12304	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12305	{
12306	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12307
12308	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
12309	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
12310	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
12311	return VERR_EM_INTERPRETER;
12312	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12313	HMVMX_RETURN_UNEXPECTED_EXIT();
12314	}
12315
12316
12317	/**
12318	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
12319	*/
12320	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12321	{
12322	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12323
12324	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
12325	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12326	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12327	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12328	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12329	{
12330	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12331	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12332	}
12333	AssertRCReturn(rc, rc);
12334	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
12335
12336	#ifdef VBOX_STRICT
12337	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
12338	{
12339	if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
12340	&& pMixedCtx->ecx != MSR_K6_EFER)
12341	{
12342	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12343	pMixedCtx->ecx));
12344	HMVMX_RETURN_UNEXPECTED_EXIT();
12345	}
12346	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12347	{
12348	VMXMSREXITREAD enmRead;
12349	VMXMSREXITWRITE enmWrite;
12350	int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12351	AssertRCReturn(rc2, rc2);
12352	if (enmRead == VMXMSREXIT_PASSTHRU_READ)
12353	{
12354	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12355	HMVMX_RETURN_UNEXPECTED_EXIT();
12356	}
12357	}
12358	}
12359	#endif
12360
12361	PVM pVM = pVCpu->CTX_SUFF(pVM);
12362	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12363	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
12364	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
12365	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
12366	if (RT_SUCCESS(rc))
12367	{
12368	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12369	Assert(pVmxTransient->cbInstr == 2);
12370	}
12371	return rc;
12372	}
12373
12374
12375	/**
12376	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
12377	*/
12378	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12379	{
12380	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12381	PVM pVM = pVCpu->CTX_SUFF(pVM);
12382	int rc = VINF_SUCCESS;
12383
12384	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
12385	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12386	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12387	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12388	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12389	{
12390	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12391	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12392	}
12393	AssertRCReturn(rc, rc);
12394	Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
12395
12396	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12397	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
12398	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12399
12400	if (RT_SUCCESS(rc))
12401	{
12402	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12403
12404	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
12405	if ( pMixedCtx->ecx == MSR_IA32_APICBASE
12406	\|\| ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
12407	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
12408	{
12409	/*
12410	* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12411	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
12412	* EMInterpretWrmsr() changes it.
12413	*/
12414	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12415	}
12416	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12417	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12418	else if (pMixedCtx->ecx == MSR_K6_EFER)
12419	{
12420	/*
12421	* If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12422	* even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12423	* the other bits as well, SCE and NXE. See @bugref{7368}.
12424	*/
12425	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_CTLS \| HM_CHANGED_VMX_EXIT_CTLS);
12426	}
12427
12428	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12429	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12430	{
12431	switch (pMixedCtx->ecx)
12432	{
12433	/*
12434	* For SYSENTER CS, EIP, ESP MSRs, we set both the flags here so we don't accidentally
12435	* overwrite the changed guest-CPU context value while going to ring-3, see @bufref{8745}.
12436	*/
12437	case MSR_IA32_SYSENTER_CS:
12438	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
12439	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
12440	break;
12441	case MSR_IA32_SYSENTER_EIP:
12442	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
12443	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
12444	break;
12445	case MSR_IA32_SYSENTER_ESP:
12446	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
12447	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
12448	break;
12449	case MSR_K8_FS_BASE: RT_FALL_THRU();
12450	case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
12451	case MSR_K6_EFER: /* already handled above */ break;
12452	default:
12453	{
12454	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12455	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12456	else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12457	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
12458	break;
12459	}
12460	}
12461	}
12462	#ifdef VBOX_STRICT
12463	else
12464	{
12465	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12466	switch (pMixedCtx->ecx)
12467	{
12468	case MSR_IA32_SYSENTER_CS:
12469	case MSR_IA32_SYSENTER_EIP:
12470	case MSR_IA32_SYSENTER_ESP:
12471	case MSR_K8_FS_BASE:
12472	case MSR_K8_GS_BASE:
12473	{
12474	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12475	HMVMX_RETURN_UNEXPECTED_EXIT();
12476	}
12477
12478	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12479	default:
12480	{
12481	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12482	{
12483	/* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
12484	if (pMixedCtx->ecx != MSR_K6_EFER)
12485	{
12486	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12487	pMixedCtx->ecx));
12488	HMVMX_RETURN_UNEXPECTED_EXIT();
12489	}
12490	}
12491
12492	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12493	{
12494	VMXMSREXITREAD enmRead;
12495	VMXMSREXITWRITE enmWrite;
12496	int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12497	AssertRCReturn(rc2, rc2);
12498	if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12499	{
12500	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12501	HMVMX_RETURN_UNEXPECTED_EXIT();
12502	}
12503	}
12504	break;
12505	}
12506	}
12507	}
12508	#endif /* VBOX_STRICT */
12509	}
12510	return rc;
12511	}
12512
12513
12514	/**
12515	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12516	*/
12517	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12518	{
12519	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12520
12521	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
12522	return VINF_EM_RAW_INTERRUPT;
12523	}
12524
12525
12526	/**
12527	* VM-exit handler for when the TPR value is lowered below the specified
12528	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12529	*/
12530	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12531	{
12532	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12533	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12534
12535	/*
12536	* The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12537	* pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12538	*/
12539	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12540	return VINF_SUCCESS;
12541	}
12542
12543
12544	/**
12545	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12546	* VM-exit.
12547	*
12548	* @retval VINF_SUCCESS when guest execution can continue.
12549	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12550	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12551	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12552	* interpreter.
12553	*/
12554	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12555	{
12556	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12557	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12558	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12559	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12560	AssertRCReturn(rc, rc);
12561
12562	RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12563	uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12564	PVM pVM = pVCpu->CTX_SUFF(pVM);
12565	VBOXSTRICTRC rcStrict;
12566	rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /fMemory/, true /fNeedRsp/);
12567	switch (uAccessType)
12568	{
12569	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12570	{
12571	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12572	AssertRCReturn(rc, rc);
12573
12574	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12575	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12576	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12577	AssertMsg( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT \|\| rcStrict == VINF_PGM_CHANGE_MODE
12578	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12579	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12580	{
12581	case 0: /* CR0 */
12582	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12583	Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12584	break;
12585	case 2: /* CR2 */
12586	/* Nothing to do here, CR2 it's not part of the VMCS. */
12587	break;
12588	case 3: /* CR3 */
12589	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx) \|\| pVCpu->hm.s.fUsingDebugLoop);
12590	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12591	Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12592	break;
12593	case 4: /* CR4 */
12594	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12595	Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12596	VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12597	break;
12598	case 8: /* CR8 */
12599	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12600	/* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12601	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12602	break;
12603	default:
12604	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12605	break;
12606	}
12607
12608	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12609	break;
12610	}
12611
12612	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12613	{
12614	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12615	AssertRCReturn(rc, rc);
12616
12617	Assert( !pVM->hm.s.fNestedPaging
12618	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12619	\|\| pVCpu->hm.s.fUsingDebugLoop
12620	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12621
12622	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12623	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12624	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12625
12626	rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12627	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12628	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12629	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12630	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12631	Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12632	VBOXSTRICTRC_VAL(rcStrict)));
12633	break;
12634	}
12635
12636	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12637	{
12638	AssertRCReturn(rc, rc);
12639	rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12640	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12641	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12642	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12643	Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12644	break;
12645	}
12646
12647	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12648	{
12649	AssertRCReturn(rc, rc);
12650	rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12651	VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12652	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT \|\| rcStrict == VINF_PGM_CHANGE_MODE,
12653	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12654	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12655	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12656	Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12657	break;
12658	}
12659
12660	default:
12661	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12662	VERR_VMX_UNEXPECTED_EXCEPTION);
12663	}
12664
12665	HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12666	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12667	NOREF(pVM);
12668	return rcStrict;
12669	}
12670
12671
12672	/**
12673	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12674	* VM-exit.
12675	*/
12676	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12677	{
12678	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12679	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12680
12681	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12682	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12683	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12684	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12685	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12686	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12687	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12688	AssertRCReturn(rc, rc);
12689
12690	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12691	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12692	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12693	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12694	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12695	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12696	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12697	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12698	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12699
12700	/* I/O operation lookup arrays. */
12701	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12702	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12703
12704	VBOXSTRICTRC rcStrict;
12705	uint32_t const cbValue = s_aIOSizes[uIOWidth];
12706	uint32_t const cbInstr = pVmxTransient->cbInstr;
12707	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12708	PVM pVM = pVCpu->CTX_SUFF(pVM);
12709	if (fIOString)
12710	{
12711	#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12712	See @bugref{5752#c158}. Should work now. */
12713	/*
12714	* INS/OUTS - I/O String instruction.
12715	*
12716	* Use instruction-information if available, otherwise fall back on
12717	* interpreting the instruction.
12718	*/
12719	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12720	fIOWrite ? 'w' : 'r'));
12721	AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12722	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12723	{
12724	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12725	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12726	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12727	AssertRCReturn(rc2, rc2);
12728	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12729	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12730	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12731	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12732	if (fIOWrite)
12733	{
12734	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12735	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
12736	}
12737	else
12738	{
12739	/*
12740	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12741	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12742	* See Intel Instruction spec. for "INS".
12743	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12744	*/
12745	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
12746	}
12747	}
12748	else
12749	{
12750	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12751	int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12752	AssertRCReturn(rc2, rc2);
12753	rcStrict = IEMExecOne(pVCpu);
12754	}
12755	/** @todo IEM needs to be setting these flags somehow. */
12756	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12757	fUpdateRipAlready = true;
12758	#else
12759	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12760	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12761	if (RT_SUCCESS(rcStrict))
12762	{
12763	if (fIOWrite)
12764	{
12765	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12766	(DISCPUMODE)pDis->uAddrMode, cbValue);
12767	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12768	}
12769	else
12770	{
12771	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12772	(DISCPUMODE)pDis->uAddrMode, cbValue);
12773	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12774	}
12775	}
12776	else
12777	{
12778	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12779	pMixedCtx->rip));
12780	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12781	}
12782	#endif
12783	}
12784	else
12785	{
12786	/*
12787	* IN/OUT - I/O instruction.
12788	*/
12789	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12790	uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12791	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12792	if (fIOWrite)
12793	{
12794	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12795	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12796	}
12797	else
12798	{
12799	uint32_t u32Result = 0;
12800	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12801	if (IOM_SUCCESS(rcStrict))
12802	{
12803	/* Save result of I/O IN instr. in AL/AX/EAX. */
12804	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
12805	}
12806	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12807	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12808	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12809	}
12810	}
12811
12812	if (IOM_SUCCESS(rcStrict))
12813	{
12814	if (!fUpdateRipAlready)
12815	{
12816	hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12817	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12818	}
12819
12820	/*
12821	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12822	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12823	*/
12824	if (fIOString)
12825	{
12826	/** @todo Single-step for INS/OUTS with REP prefix? */
12827	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12828	}
12829	else if ( !fDbgStepping
12830	&& fGstStepping)
12831	{
12832	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12833	}
12834
12835	/*
12836	* If any I/O breakpoints are armed, we need to check if one triggered
12837	* and take appropriate action.
12838	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12839	*/
12840	int rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12841	AssertRCReturn(rc2, rc2);
12842
12843	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12844	* execution engines about whether hyper BPs and such are pending. */
12845	uint32_t const uDr7 = pMixedCtx->dr[7];
12846	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12847	&& X86_DR7_ANY_RW_IO(uDr7)
12848	&& (pMixedCtx->cr4 & X86_CR4_DE))
12849	\|\| DBGFBpIsHwIoArmed(pVM)))
12850	{
12851	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12852
12853	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12854	VMMRZCallRing3Disable(pVCpu);
12855	HM_DISABLE_PREEMPT();
12856
12857	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12858
12859	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12860	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12861	{
12862	/* Raise #DB. */
12863	if (fIsGuestDbgActive)
12864	ASMSetDR6(pMixedCtx->dr[6]);
12865	if (pMixedCtx->dr[7] != uDr7)
12866	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12867
12868	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12869	}
12870	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12871	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12872	else if ( rcStrict2 != VINF_SUCCESS
12873	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
12874	rcStrict = rcStrict2;
12875	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12876
12877	HM_RESTORE_PREEMPT();
12878	VMMRZCallRing3Enable(pVCpu);
12879	}
12880	}
12881
12882	#ifdef VBOX_STRICT
12883	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12884	Assert(!fIOWrite);
12885	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE \|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12886	Assert(fIOWrite);
12887	else
12888	{
12889	#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12890	* statuses, that the VMM device and some others may return. See
12891	* IOM_SUCCESS() for guidance. */
12892	AssertMsg( RT_FAILURE(rcStrict)
12893	\|\| rcStrict == VINF_SUCCESS
12894	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
12895	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
12896	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
12897	\|\| rcStrict == VINF_EM_RAW_TO_R3
12898	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12899	#endif
12900	}
12901	#endif
12902
12903	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12904	return rcStrict;
12905	}
12906
12907
12908	/**
12909	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12910	* VM-exit.
12911	*/
12912	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12913	{
12914	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12915
12916	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
12917	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12918	AssertRCReturn(rc, rc);
12919	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12920	{
12921	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12922	AssertRCReturn(rc, rc);
12923	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12924	{
12925	uint32_t uErrCode;
12926	RTGCUINTPTR GCPtrFaultAddress;
12927	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12928	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12929	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12930	if (fErrorCodeValid)
12931	{
12932	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12933	AssertRCReturn(rc, rc);
12934	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12935	}
12936	else
12937	uErrCode = 0;
12938
12939	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12940	&& uVector == X86_XCPT_PF)
12941	GCPtrFaultAddress = pMixedCtx->cr2;
12942	else
12943	GCPtrFaultAddress = 0;
12944
12945	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12946	0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
12947
12948	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12949	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12950	return VINF_EM_RAW_INJECT_TRPM_EVENT;
12951	}
12952	}
12953
12954	/* Fall back to the interpreter to emulate the task-switch. */
12955	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12956	return VERR_EM_INTERPRETER;
12957	}
12958
12959
12960	/**
12961	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12962	*/
12963	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12964	{
12965	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12966	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12967	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12968	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12969	AssertRCReturn(rc, rc);
12970	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12971	return VINF_EM_DBG_STEPPED;
12972	}
12973
12974
12975	/**
12976	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12977	*/
12978	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12979	{
12980	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12981
12982	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12983
12984	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12985	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12986	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12987	{
12988	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12989	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12990	{
12991	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12992	return VINF_EM_RAW_INJECT_TRPM_EVENT;
12993	}
12994	}
12995	else
12996	{
12997	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12998	rcStrict1 = VINF_SUCCESS;
12999	return rcStrict1;
13000	}
13001
13002	#if 0
13003	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
13004	* just sync the whole thing. */
13005	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13006	#else
13007	/* Aggressive state sync. for now. */
13008	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13009	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13010	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13011	#endif
13012	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13013	AssertRCReturn(rc, rc);
13014
13015	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
13016	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
13017	VBOXSTRICTRC rcStrict2;
13018	switch (uAccessType)
13019	{
13020	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
13021	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
13022	{
13023	AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
13024	\|\| VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
13025	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
13026
13027	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
13028	GCPhys &= PAGE_BASE_GC_MASK;
13029	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
13030	PVM pVM = pVCpu->CTX_SUFF(pVM);
13031	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
13032	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
13033
13034	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
13035	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
13036	CPUMCTX2CORE(pMixedCtx), GCPhys);
13037	Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
13038	if ( rcStrict2 == VINF_SUCCESS
13039	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13040	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
13041	{
13042	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13043	\| HM_CHANGED_GUEST_RSP
13044	\| HM_CHANGED_GUEST_RFLAGS
13045	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
13046	rcStrict2 = VINF_SUCCESS;
13047	}
13048	break;
13049	}
13050
13051	default:
13052	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
13053	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
13054	break;
13055	}
13056
13057	if (rcStrict2 != VINF_SUCCESS)
13058	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
13059	return rcStrict2;
13060	}
13061
13062
13063	/**
13064	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
13065	* VM-exit.
13066	*/
13067	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13068	{
13069	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13070
13071	/* We should -not- get this VM-exit if the guest's debug registers were active. */
13072	if (pVmxTransient->fWasGuestDebugStateActive)
13073	{
13074	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
13075	HMVMX_RETURN_UNEXPECTED_EXIT();
13076	}
13077
13078	if ( !pVCpu->hm.s.fSingleInstruction
13079	&& !pVmxTransient->fWasHyperDebugStateActive)
13080	{
13081	Assert(!DBGFIsStepping(pVCpu));
13082	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
13083
13084	/* Don't intercept MOV DRx any more. */
13085	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
13086	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
13087	AssertRCReturn(rc, rc);
13088
13089	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
13090	VMMRZCallRing3Disable(pVCpu);
13091	HM_DISABLE_PREEMPT();
13092
13093	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
13094	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
13095	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
13096
13097	HM_RESTORE_PREEMPT();
13098	VMMRZCallRing3Enable(pVCpu);
13099
13100	#ifdef VBOX_WITH_STATISTICS
13101	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13102	AssertRCReturn(rc, rc);
13103	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
13104	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
13105	else
13106	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
13107	#endif
13108	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
13109	return VINF_SUCCESS;
13110	}
13111
13112	/*
13113	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
13114	* Update the segment registers and DR7 from the CPU.
13115	*/
13116	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13117	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13118	rc \|= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13119	AssertRCReturn(rc, rc);
13120	Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13121
13122	PVM pVM = pVCpu->CTX_SUFF(pVM);
13123	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
13124	{
13125	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
13126	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
13127	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
13128	if (RT_SUCCESS(rc))
13129	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
13130	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
13131	}
13132	else
13133	{
13134	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
13135	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
13136	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
13137	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
13138	}
13139
13140	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
13141	if (RT_SUCCESS(rc))
13142	{
13143	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13144	AssertRCReturn(rc2, rc2);
13145	return VINF_SUCCESS;
13146	}
13147	return rc;
13148	}
13149
13150
13151	/**
13152	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
13153	* Conditional VM-exit.
13154	*/
13155	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13156	{
13157	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13158	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
13159
13160	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13161	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13162	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13163	{
13164	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
13165	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
13166	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13167	{
13168	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
13169	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13170	}
13171	}
13172	else
13173	{
13174	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13175	rcStrict1 = VINF_SUCCESS;
13176	return rcStrict1;
13177	}
13178
13179	RTGCPHYS GCPhys = 0;
13180	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
13181
13182	#if 0
13183	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
13184	#else
13185	/* Aggressive state sync. for now. */
13186	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13187	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13188	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13189	#endif
13190	AssertRCReturn(rc, rc);
13191
13192	/*
13193	* If we succeed, resume guest execution.
13194	* If we fail in interpreting the instruction because we couldn't get the guest physical address
13195	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
13196	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
13197	* weird case. See @bugref{6043}.
13198	*/
13199	PVM pVM = pVCpu->CTX_SUFF(pVM);
13200	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
13201	Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
13202	if ( rcStrict2 == VINF_SUCCESS
13203	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13204	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
13205	{
13206	/* Successfully handled MMIO operation. */
13207	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13208	\| HM_CHANGED_GUEST_RSP
13209	\| HM_CHANGED_GUEST_RFLAGS
13210	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
13211	return VINF_SUCCESS;
13212	}
13213	return rcStrict2;
13214	}
13215
13216
13217	/**
13218	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
13219	* VM-exit.
13220	*/
13221	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13222	{
13223	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13224	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
13225
13226	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13227	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13228	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13229	{
13230	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
13231	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13232	Log4(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
13233	}
13234	else
13235	{
13236	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13237	rcStrict1 = VINF_SUCCESS;
13238	return rcStrict1;
13239	}
13240
13241	RTGCPHYS GCPhys = 0;
13242	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
13243	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13244	#if 0
13245	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
13246	#else
13247	/* Aggressive state sync. for now. */
13248	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13249	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13250	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13251	#endif
13252	AssertRCReturn(rc, rc);
13253
13254	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
13255	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
13256
13257	RTGCUINT uErrorCode = 0;
13258	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
13259	uErrorCode \|= X86_TRAP_PF_ID;
13260	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
13261	uErrorCode \|= X86_TRAP_PF_RW;
13262	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
13263	uErrorCode \|= X86_TRAP_PF_P;
13264
13265	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
13266
13267	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
13268	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13269
13270	/* Handle the pagefault trap for the nested shadow table. */
13271	PVM pVM = pVCpu->CTX_SUFF(pVM);
13272	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
13273	TRPMResetTrap(pVCpu);
13274
13275	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
13276	if ( rcStrict2 == VINF_SUCCESS
13277	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13278	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
13279	{
13280	/* Successfully synced our nested page tables. */
13281	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
13282	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13283	\| HM_CHANGED_GUEST_RSP
13284	\| HM_CHANGED_GUEST_RFLAGS);
13285	return VINF_SUCCESS;
13286	}
13287
13288	Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
13289	return rcStrict2;
13290	}
13291
13292	/** @} */
13293
13294	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13295	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
13296	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13297
13298	/** @name VM-exit exception handlers.
13299	* @{
13300	*/
13301
13302	/**
13303	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
13304	*/
13305	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13306	{
13307	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13308	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13309
13310	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13311	AssertRCReturn(rc, rc);
13312
13313	if (!(pMixedCtx->cr0 & X86_CR0_NE))
13314	{
13315	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13316	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13317
13318	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13319	* provides VM-exit instruction length. If this causes problem later,
13320	* disassemble the instruction like it's done on AMD-V. */
13321	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13322	AssertRCReturn(rc2, rc2);
13323	return rc;
13324	}
13325
13326	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13327	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13328	return rc;
13329	}
13330
13331
13332	/**
13333	* VM-exit exception handler for \#BP (Breakpoint exception).
13334	*/
13335	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13336	{
13337	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13338	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13339
13340	/** @todo Try optimize this by not saving the entire guest state unless
13341	* really needed. */
13342	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13343	AssertRCReturn(rc, rc);
13344
13345	PVM pVM = pVCpu->CTX_SUFF(pVM);
13346	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
13347	if (rc == VINF_EM_RAW_GUEST_TRAP)
13348	{
13349	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13350	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13351	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13352	AssertRCReturn(rc, rc);
13353
13354	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13355	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13356	}
13357
13358	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
13359	return rc;
13360	}
13361
13362
13363	/**
13364	* VM-exit exception handler for \#AC (alignment check exception).
13365	*/
13366	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13367	{
13368	RT_NOREF_PV(pMixedCtx);
13369	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13370
13371	/*
13372	* Re-inject it. We'll detect any nesting before getting here.
13373	*/
13374	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13375	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13376	AssertRCReturn(rc, rc);
13377	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13378
13379	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13380	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13381	return VINF_SUCCESS;
13382	}
13383
13384
13385	/**
13386	* VM-exit exception handler for \#DB (Debug exception).
13387	*/
13388	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13389	{
13390	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13391	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13392	Log6(("XcptDB\n"));
13393
13394	/*
13395	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13396	* for processing.
13397	*/
13398	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13399	AssertRCReturn(rc, rc);
13400
13401	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13402	uint64_t uDR6 = X86_DR6_INIT_VAL;
13403	uDR6 \|= ( pVmxTransient->uExitQualification
13404	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
13405
13406	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13407	if (rc == VINF_EM_RAW_GUEST_TRAP)
13408	{
13409	/*
13410	* The exception was for the guest. Update DR6, DR7.GD and
13411	* IA32_DEBUGCTL.LBR before forwarding it.
13412	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13413	*/
13414	VMMRZCallRing3Disable(pVCpu);
13415	HM_DISABLE_PREEMPT();
13416
13417	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
13418	pMixedCtx->dr[6] \|= uDR6;
13419	if (CPUMIsGuestDebugStateActive(pVCpu))
13420	ASMSetDR6(pMixedCtx->dr[6]);
13421
13422	HM_RESTORE_PREEMPT();
13423	VMMRZCallRing3Enable(pVCpu);
13424
13425	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13426	AssertRCReturn(rc, rc);
13427
13428	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13429	pMixedCtx->dr[7] &= ~X86_DR7_GD;
13430
13431	/* Paranoia. */
13432	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13433	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
13434
13435	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
13436	AssertRCReturn(rc, rc);
13437
13438	/*
13439	* Raise #DB in the guest.
13440	*
13441	* It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
13442	* hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
13443	* Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
13444	*
13445	* Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
13446	*/
13447	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13448	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13449	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13450	AssertRCReturn(rc, rc);
13451	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13452	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13453	return VINF_SUCCESS;
13454	}
13455
13456	/*
13457	* Not a guest trap, must be a hypervisor related debug event then.
13458	* Update DR6 in case someone is interested in it.
13459	*/
13460	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13461	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13462	CPUMSetHyperDR6(pVCpu, uDR6);
13463
13464	return rc;
13465	}
13466
13467
13468	/**
13469	* VM-exit exception handler for \#NM (Device-not-available exception: floating
13470	* point exception).
13471	*/
13472	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13473	{
13474	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13475
13476	/* We require CR0 and EFER. EFER is always up-to-date. */
13477	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13478	AssertRCReturn(rc, rc);
13479
13480	/* We're playing with the host CPU state here, have to disable preemption or longjmp. */
13481	VMMRZCallRing3Disable(pVCpu);
13482	HM_DISABLE_PREEMPT();
13483
13484	/* If the guest FPU was active at the time of the #NM VM-exit, then it's a guest fault. */
13485	if (pVmxTransient->fWasGuestFPUStateActive)
13486	{
13487	rc = VINF_EM_RAW_GUEST_TRAP;
13488	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
13489	}
13490	else
13491	{
13492	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13493	Assert(!pVmxTransient->fWasGuestFPUStateActive \|\| pVCpu->hm.s.fUsingDebugLoop);
13494	#endif
13495	rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu);
13496	Assert( rc == VINF_EM_RAW_GUEST_TRAP
13497	\|\| ((rc == VINF_SUCCESS \|\| rc == VINF_CPUM_HOST_CR0_MODIFIED) && CPUMIsGuestFPUStateActive(pVCpu)));
13498	if (rc == VINF_CPUM_HOST_CR0_MODIFIED)
13499	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
13500	}
13501
13502	HM_RESTORE_PREEMPT();
13503	VMMRZCallRing3Enable(pVCpu);
13504
13505	if (rc == VINF_SUCCESS \|\| rc == VINF_CPUM_HOST_CR0_MODIFIED)
13506	{
13507	/* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
13508	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
13509	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
13510	pVCpu->hm.s.fPreloadGuestFpu = true;
13511	}
13512	else
13513	{
13514	/* Forward #NM to the guest. */
13515	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
13516	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13517	AssertRCReturn(rc, rc);
13518	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13519	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
13520	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13521	}
13522
13523	return VINF_SUCCESS;
13524	}
13525
13526
13527	/**
13528	* VM-exit exception handler for \#GP (General-protection exception).
13529	*
13530	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13531	*/
13532	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13533	{
13534	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13535	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13536
13537	int rc;
13538	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13539	{ /* likely */ }
13540	else
13541	{
13542	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13543	Assert(pVCpu->hm.s.fUsingDebugLoop);
13544	#endif
13545	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13546	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13547	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13548	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13549	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13550	AssertRCReturn(rc, rc);
13551	Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13552	pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13553	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13554	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13555	return rc;
13556	}
13557
13558	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13559	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13560
13561	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13562	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13563	AssertRCReturn(rc, rc);
13564
13565	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13566	uint32_t cbOp = 0;
13567	PVM pVM = pVCpu->CTX_SUFF(pVM);
13568	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13569	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13570	if (RT_SUCCESS(rc))
13571	{
13572	rc = VINF_SUCCESS;
13573	Assert(cbOp == pDis->cbInstr);
13574	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13575	switch (pDis->pCurInstr->uOpcode)
13576	{
13577	case OP_CLI:
13578	{
13579	pMixedCtx->eflags.Bits.u1IF = 0;
13580	pMixedCtx->eflags.Bits.u1RF = 0;
13581	pMixedCtx->rip += pDis->cbInstr;
13582	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13583	if ( !fDbgStepping
13584	&& pMixedCtx->eflags.Bits.u1TF)
13585	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13586	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13587	break;
13588	}
13589
13590	case OP_STI:
13591	{
13592	bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13593	pMixedCtx->eflags.Bits.u1IF = 1;
13594	pMixedCtx->eflags.Bits.u1RF = 0;
13595	pMixedCtx->rip += pDis->cbInstr;
13596	if (!fOldIF)
13597	{
13598	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13599	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13600	}
13601	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13602	if ( !fDbgStepping
13603	&& pMixedCtx->eflags.Bits.u1TF)
13604	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13605	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13606	break;
13607	}
13608
13609	case OP_HLT:
13610	{
13611	rc = VINF_EM_HALT;
13612	pMixedCtx->rip += pDis->cbInstr;
13613	pMixedCtx->eflags.Bits.u1RF = 0;
13614	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13615	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13616	break;
13617	}
13618
13619	case OP_POPF:
13620	{
13621	Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13622	uint32_t cbParm;
13623	uint32_t uMask;
13624	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13625	if (pDis->fPrefix & DISPREFIX_OPSIZE)
13626	{
13627	cbParm = 4;
13628	uMask = 0xffffffff;
13629	}
13630	else
13631	{
13632	cbParm = 2;
13633	uMask = 0xffff;
13634	}
13635
13636	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
13637	RTGCPTR GCPtrStack = 0;
13638	X86EFLAGS Eflags;
13639	Eflags.u32 = 0;
13640	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13641	&GCPtrStack);
13642	if (RT_SUCCESS(rc))
13643	{
13644	Assert(sizeof(Eflags.u32) >= cbParm);
13645	rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13646	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13647	}
13648	if (RT_FAILURE(rc))
13649	{
13650	rc = VERR_EM_INTERPRETER;
13651	break;
13652	}
13653	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13654	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) \| X86_EFL_RF))
13655	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13656	pMixedCtx->esp += cbParm;
13657	pMixedCtx->esp &= uMask;
13658	pMixedCtx->rip += pDis->cbInstr;
13659	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13660	\| HM_CHANGED_GUEST_RSP
13661	\| HM_CHANGED_GUEST_RFLAGS);
13662	/* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13663	POPF restores EFLAGS.TF. */
13664	if ( !fDbgStepping
13665	&& fGstStepping)
13666	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13667	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13668	break;
13669	}
13670
13671	case OP_PUSHF:
13672	{
13673	uint32_t cbParm;
13674	uint32_t uMask;
13675	if (pDis->fPrefix & DISPREFIX_OPSIZE)
13676	{
13677	cbParm = 4;
13678	uMask = 0xffffffff;
13679	}
13680	else
13681	{
13682	cbParm = 2;
13683	uMask = 0xffff;
13684	}
13685
13686	/* Get the stack pointer & push the contents of eflags onto the stack. */
13687	RTGCPTR GCPtrStack = 0;
13688	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13689	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13690	if (RT_FAILURE(rc))
13691	{
13692	rc = VERR_EM_INTERPRETER;
13693	break;
13694	}
13695	X86EFLAGS Eflags = pMixedCtx->eflags;
13696	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13697	Eflags.Bits.u1RF = 0;
13698	Eflags.Bits.u1VM = 0;
13699
13700	rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13701	if (RT_UNLIKELY(rc != VINF_SUCCESS))
13702	{
13703	AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13704	rc = VERR_EM_INTERPRETER;
13705	break;
13706	}
13707	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13708	pMixedCtx->esp -= cbParm;
13709	pMixedCtx->esp &= uMask;
13710	pMixedCtx->rip += pDis->cbInstr;
13711	pMixedCtx->eflags.Bits.u1RF = 0;
13712	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13713	\| HM_CHANGED_GUEST_RSP
13714	\| HM_CHANGED_GUEST_RFLAGS);
13715	if ( !fDbgStepping
13716	&& pMixedCtx->eflags.Bits.u1TF)
13717	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13718	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13719	break;
13720	}
13721
13722	case OP_IRET:
13723	{
13724	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13725	* instruction reference. */
13726	RTGCPTR GCPtrStack = 0;
13727	uint32_t uMask = 0xffff;
13728	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13729	uint16_t aIretFrame[3];
13730	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
13731	{
13732	rc = VERR_EM_INTERPRETER;
13733	break;
13734	}
13735	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13736	&GCPtrStack);
13737	if (RT_SUCCESS(rc))
13738	{
13739	rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13740	PGMACCESSORIGIN_HM));
13741	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13742	}
13743	if (RT_FAILURE(rc))
13744	{
13745	rc = VERR_EM_INTERPRETER;
13746	break;
13747	}
13748	pMixedCtx->eip = 0;
13749	pMixedCtx->ip = aIretFrame[0];
13750	pMixedCtx->cs.Sel = aIretFrame[1];
13751	pMixedCtx->cs.ValidSel = aIretFrame[1];
13752	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13753	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) \| X86_EFL_1) & ~X86_EFL_RF))
13754	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13755	pMixedCtx->sp += sizeof(aIretFrame);
13756	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13757	\| HM_CHANGED_GUEST_SEGMENT_REGS
13758	\| HM_CHANGED_GUEST_RSP
13759	\| HM_CHANGED_GUEST_RFLAGS);
13760	/* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13761	if ( !fDbgStepping
13762	&& fGstStepping)
13763	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13764	Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13765	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13766	break;
13767	}
13768
13769	case OP_INT:
13770	{
13771	uint16_t uVector = pDis->Param1.uValue & 0xff;
13772	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13773	/* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13774	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13775	break;
13776	}
13777
13778	case OP_INTO:
13779	{
13780	if (pMixedCtx->eflags.Bits.u1OF)
13781	{
13782	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13783	/* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13784	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13785	}
13786	else
13787	{
13788	pMixedCtx->eflags.Bits.u1RF = 0;
13789	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13790	}
13791	break;
13792	}
13793
13794	default:
13795	{
13796	pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13797	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13798	EMCODETYPE_SUPERVISOR);
13799	rc = VBOXSTRICTRC_VAL(rc2);
13800	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13801	/** @todo We have to set pending-debug exceptions here when the guest is
13802	* single-stepping depending on the instruction that was interpreted. */
13803	Log4(("#GP rc=%Rrc\n", rc));
13804	break;
13805	}
13806	}
13807	}
13808	else
13809	rc = VERR_EM_INTERPRETER;
13810
13811	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
13812	("#GP Unexpected rc=%Rrc\n", rc));
13813	return rc;
13814	}
13815
13816
13817	/**
13818	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13819	* the exception reported in the VMX transient structure back into the VM.
13820	*
13821	* @remarks Requires uExitIntInfo in the VMX transient structure to be
13822	* up-to-date.
13823	*/
13824	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13825	{
13826	RT_NOREF_PV(pMixedCtx);
13827	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13828	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13829	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13830	("uVector=%#04x u32XcptBitmap=%#010RX32\n",
13831	VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13832	#endif
13833
13834	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13835	hmR0VmxCheckExitDueToEventDelivery(). */
13836	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13837	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13838	AssertRCReturn(rc, rc);
13839	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13840
13841	#ifdef DEBUG_ramshankar
13842	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13843	uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13844	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13845	#endif
13846
13847	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13848	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13849	return VINF_SUCCESS;
13850	}
13851
13852
13853	/**
13854	* VM-exit exception handler for \#PF (Page-fault exception).
13855	*/
13856	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13857	{
13858	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13859	PVM pVM = pVCpu->CTX_SUFF(pVM);
13860	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13861	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13862	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13863	AssertRCReturn(rc, rc);
13864
13865	if (!pVM->hm.s.fNestedPaging)
13866	{ /* likely */ }
13867	else
13868	{
13869	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13870	Assert(pVCpu->hm.s.fUsingDebugLoop);
13871	#endif
13872	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13873	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13874	{
13875	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13876	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13877	}
13878	else
13879	{
13880	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13881	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13882	Log4(("Pending #DF due to vectoring #PF. NP\n"));
13883	}
13884	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13885	return rc;
13886	}
13887
13888	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13889	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13890	if (pVmxTransient->fVectoringPF)
13891	{
13892	Assert(pVCpu->hm.s.Event.fPending);
13893	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13894	}
13895
13896	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13897	AssertRCReturn(rc, rc);
13898
13899	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13900	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13901
13902	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13903	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13904	(RTGCPTR)pVmxTransient->uExitQualification);
13905
13906	Log4(("#PF: rc=%Rrc\n", rc));
13907	if (rc == VINF_SUCCESS)
13908	{
13909	#if 0
13910	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13911	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
13912	* memory? We don't update the whole state here... */
13913	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13914	\| HM_CHANGED_GUEST_RSP
13915	\| HM_CHANGED_GUEST_RFLAGS
13916	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
13917	#else
13918	/*
13919	* This is typically a shadow page table sync or a MMIO instruction. But we may have
13920	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13921	*/
13922	/** @todo take advantage of CPUM changed flags instead of brute forcing. */
13923	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13924	#endif
13925	TRPMResetTrap(pVCpu);
13926	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13927	return rc;
13928	}
13929
13930	if (rc == VINF_EM_RAW_GUEST_TRAP)
13931	{
13932	if (!pVmxTransient->fVectoringDoublePF)
13933	{
13934	/* It's a guest page fault and needs to be reflected to the guest. */
13935	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13936	TRPMResetTrap(pVCpu);
13937	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13938	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13939	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13940	}
13941	else
13942	{
13943	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13944	TRPMResetTrap(pVCpu);
13945	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13946	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13947	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
13948	}
13949
13950	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13951	return VINF_SUCCESS;
13952	}
13953
13954	TRPMResetTrap(pVCpu);
13955	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13956	return rc;
13957	}
13958
13959	/** @} */
13960

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 70161

Download in other formats: