HMVMXR0.cpp@ 80090

Last change on this file since 80090 was 80080, checked in by vboxsync, 6 years ago
VMM: Kicking out raw-mode and 32-bit hosts - HM, VMMSWITCHER, ++. bugref:9517 bugref:9511
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 714.2 KB

Line
1	/* $Id: HMVMXR0.cpp 80080 2019-07-31 16:12:31Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27	#include <iprt/mem.h>
28	#include <iprt/mp.h>
29
30	#include <VBox/vmm/pdmapi.h>
31	#include <VBox/vmm/dbgf.h>
32	#include <VBox/vmm/iem.h>
33	#include <VBox/vmm/iom.h>
34	#include <VBox/vmm/selm.h>
35	#include <VBox/vmm/tm.h>
36	#include <VBox/vmm/em.h>
37	#include <VBox/vmm/gim.h>
38	#include <VBox/vmm/apic.h>
39	#ifdef VBOX_WITH_REM
40	# include <VBox/vmm/rem.h>
41	#endif
42	#include "HMInternal.h"
43	#include <VBox/vmm/vm.h>
44	#include <VBox/vmm/hmvmxinline.h>
45	#include "HMVMXR0.h"
46	#include "dtrace/VBoxVMM.h"
47
48	#ifdef DEBUG_ramshankar
49	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
50	# define HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
51	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
52	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
53	# define HMVMX_ALWAYS_CLEAN_TRANSIENT
54	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
55	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
56	# define HMVMX_ALWAYS_TRAP_PF
57	# define HMVMX_ALWAYS_FLUSH_TLB
58	# define HMVMX_ALWAYS_SWAP_EFER
59	#endif
60
61
62	/*********************************************************************************************************************************
63	* Defined Constants And Macros *
64	*********************************************************************************************************************************/
65	/** Use the function table. */
66	#define HMVMX_USE_FUNCTION_TABLE
67
68	/** Determine which tagged-TLB flush handler to use. */
69	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
70	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
71	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
72	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
73
74	/**
75	* Flags to skip redundant reads of some common VMCS fields that are not part of
76	* the guest-CPU or VCPU state but are needed while handling VM-exits.
77	*/
78	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
79	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
80	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
81	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
82	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
83	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
84	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
85	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
86	#define HMVMX_READ_GUEST_PHYSICAL_ADDR RT_BIT_32(8)
87
88	/** All the VMCS fields required for processing of exception/NMI VM-exits. */
89	#define HMVMX_READ_XCPT_INFO ( HMVMX_READ_EXIT_INTERRUPTION_INFO \
90	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE \
91	\| HMVMX_READ_EXIT_INSTR_LEN \
92	\| HMVMX_READ_IDT_VECTORING_INFO \
93	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE)
94
95	/** Assert that all the given fields have been read from the VMCS. */
96	#ifdef VBOX_STRICT
97	# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) \
98	do { \
99	uint32_t const fVmcsFieldRead = ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead); \
100	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE(); \
101	Assert((fVmcsFieldRead & (a_fReadFields)) == (a_fReadFields)); \
102	} while (0)
103	#else
104	# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) do { } while (0)
105	#endif
106
107	/**
108	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
109	* guest using hardware-assisted VMX.
110	*
111	* This excludes state like GPRs (other than RSP) which are always are
112	* swapped and restored across the world-switch and also registers like EFER,
113	* MSR which cannot be modified by the guest without causing a VM-exit.
114	*/
115	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
116	\| CPUMCTX_EXTRN_RFLAGS \
117	\| CPUMCTX_EXTRN_RSP \
118	\| CPUMCTX_EXTRN_SREG_MASK \
119	\| CPUMCTX_EXTRN_TABLE_MASK \
120	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
121	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
122	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
123	\| CPUMCTX_EXTRN_TSC_AUX \
124	\| CPUMCTX_EXTRN_OTHER_MSRS \
125	\| CPUMCTX_EXTRN_CR0 \
126	\| CPUMCTX_EXTRN_CR3 \
127	\| CPUMCTX_EXTRN_CR4 \
128	\| CPUMCTX_EXTRN_DR7 \
129	\| CPUMCTX_EXTRN_HWVIRT \
130	\| CPUMCTX_EXTRN_HM_VMX_MASK)
131
132	/**
133	* Exception bitmap mask for real-mode guests (real-on-v86).
134	*
135	* We need to intercept all exceptions manually except:
136	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
137	* due to bugs in Intel CPUs.
138	* - \#PF need not be intercepted even in real-mode if we have nested paging
139	* support.
140	*/
141	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
142	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
143	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
144	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
145	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
146	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
147	\| RT_BIT(X86_XCPT_XF))
148
149	/** Maximum VM-instruction error number. */
150	#define HMVMX_INSTR_ERROR_MAX 28
151
152	/** Profiling macro. */
153	#ifdef HM_PROFILE_EXIT_DISPATCH
154	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
155	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
156	#else
157	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
158	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
159	#endif
160
161	/** Assert that preemption is disabled or covered by thread-context hooks. */
162	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
163	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
164
165	/** Assert that we haven't migrated CPUs when thread-context hooks are not
166	* used. */
167	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
168	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
169	("Illegal migration! Entered on CPU %u Current %u\n", \
170	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
171
172	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
173	* context. */
174	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
175	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
176	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
177
178	/** Log the VM-exit reason with an easily visible marker to identify it in a
179	* potential sea of logging data. */
180	#define HMVMX_LOG_EXIT(a_pVCpu, a_uExitReason) \
181	do { \
182	Log4(("VM-exit: vcpu[%RU32] %85s -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (a_pVCpu)->idCpu, \
183	HMGetVmxExitName(a_uExitReason))); \
184	} while (0) \
185
186
187	/*********************************************************************************************************************************
188	* Structures and Typedefs *
189	*********************************************************************************************************************************/
190	/**
191	* VMX per-VCPU transient state.
192	*
193	* A state structure for holding miscellaneous information across
194	* VMX non-root operation and restored after the transition.
195	*/
196	typedef struct VMXTRANSIENT
197	{
198	/** The host's rflags/eflags. */
199	RTCCUINTREG fEFlags;
200	#if HC_ARCH_BITS == 32
201	uint32_t u32Alignment0;
202	#endif
203	/** The guest's TPR value used for TPR shadowing. */
204	uint8_t u8GuestTpr;
205	/** Alignment. */
206	uint8_t abAlignment0[7];
207
208	/** The basic VM-exit reason. */
209	uint16_t uExitReason;
210	/** Alignment. */
211	uint16_t u16Alignment0;
212	/** The VM-exit interruption error code. */
213	uint32_t uExitIntErrorCode;
214	/** The VM-exit exit code qualification. */
215	uint64_t uExitQual;
216	/** The Guest-linear address. */
217	uint64_t uGuestLinearAddr;
218	/** The Guest-physical address. */
219	uint64_t uGuestPhysicalAddr;
220
221	/** The VM-exit interruption-information field. */
222	uint32_t uExitIntInfo;
223	/** The VM-exit instruction-length field. */
224	uint32_t cbInstr;
225	/** The VM-exit instruction-information field. */
226	VMXEXITINSTRINFO ExitInstrInfo;
227	/** Whether the VM-entry failed or not. */
228	bool fVMEntryFailed;
229	/** Whether we are currently executing a nested-guest. */
230	bool fIsNestedGuest;
231	/** Alignment. */
232	uint8_t abAlignment1[2];
233
234	/** The VM-entry interruption-information field. */
235	uint32_t uEntryIntInfo;
236	/** The VM-entry exception error code field. */
237	uint32_t uEntryXcptErrorCode;
238	/** The VM-entry instruction length field. */
239	uint32_t cbEntryInstr;
240
241	/** IDT-vectoring information field. */
242	uint32_t uIdtVectoringInfo;
243	/** IDT-vectoring error code. */
244	uint32_t uIdtVectoringErrorCode;
245
246	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
247	uint32_t fVmcsFieldsRead;
248
249	/** Whether the guest debug state was active at the time of VM-exit. */
250	bool fWasGuestDebugStateActive;
251	/** Whether the hyper debug state was active at the time of VM-exit. */
252	bool fWasHyperDebugStateActive;
253	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
254	bool fUpdatedTscOffsettingAndPreemptTimer;
255	/** Whether the VM-exit was caused by a page-fault during delivery of a
256	* contributory exception or a page-fault. */
257	bool fVectoringDoublePF;
258	/** Whether the VM-exit was caused by a page-fault during delivery of an
259	* external interrupt or NMI. */
260	bool fVectoringPF;
261	/** Whether the TSC_AUX MSR needs to be removed from the auto-load/store MSR
262	* area after VM-exit. */
263	bool fRemoveTscAuxMsr;
264	bool afAlignment0[2];
265
266	/** The VMCS info. object. */
267	PVMXVMCSINFO pVmcsInfo;
268	} VMXTRANSIENT;
269	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
270	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
271	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
272	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
273	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
274	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
275	/** Pointer to VMX transient state. */
276	typedef VMXTRANSIENT *PVMXTRANSIENT;
277	/** Pointer to a const VMX transient state. */
278	typedef const VMXTRANSIENT *PCVMXTRANSIENT;
279
280	/**
281	* Memory operand read or write access.
282	*/
283	typedef enum VMXMEMACCESS
284	{
285	VMXMEMACCESS_READ = 0,
286	VMXMEMACCESS_WRITE = 1
287	} VMXMEMACCESS;
288
289	/**
290	* VMX VM-exit handler.
291	*
292	* @returns Strict VBox status code (i.e. informational status codes too).
293	* @param pVCpu The cross context virtual CPU structure.
294	* @param pVmxTransient The VMX-transient structure.
295	*/
296	#ifndef HMVMX_USE_FUNCTION_TABLE
297	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
298	#else
299	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
300	/** Pointer to VM-exit handler. */
301	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
302	#endif
303
304	/**
305	* VMX VM-exit handler, non-strict status code.
306	*
307	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
308	*
309	* @returns VBox status code, no informational status code returned.
310	* @param pVCpu The cross context virtual CPU structure.
311	* @param pVmxTransient The VMX-transient structure.
312	*
313	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
314	* use of that status code will be replaced with VINF_EM_SOMETHING
315	* later when switching over to IEM.
316	*/
317	#ifndef HMVMX_USE_FUNCTION_TABLE
318	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
319	#else
320	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
321	#endif
322
323
324	/*********************************************************************************************************************************
325	* Internal Functions *
326	*********************************************************************************************************************************/
327	#ifndef HMVMX_USE_FUNCTION_TABLE
328	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
329	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
330	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
331	#else
332	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
333	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
334	#endif
335	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
336	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
337	#endif
338
339	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
340
341	/** @name VM-exit handler prototypes.
342	* @{
343	*/
344	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
345	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
346	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
347	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
348	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
349	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
350	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
351	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
352	static FNVMXEXITHANDLER hmR0VmxExitHlt;
353	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
354	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
355	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
356	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
357	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
358	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
359	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
360	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
361	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
362	static FNVMXEXITHANDLER hmR0VmxExitVmread;
363	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
364	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
365	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
366	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
367	static FNVMXEXITHANDLER hmR0VmxExitInvvpid;
368	#endif
369	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
370	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
371	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
372	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
373	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
374	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
375	static FNVMXEXITHANDLER hmR0VmxExitMwait;
376	static FNVMXEXITHANDLER hmR0VmxExitMtf;
377	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
378	static FNVMXEXITHANDLER hmR0VmxExitPause;
379	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
380	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
381	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
382	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
383	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
384	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
385	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
386	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
387	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
388	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
389	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
390	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUnexpected;
391	/** @} */
392
393	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
394	/** @name Nested-guest VM-exit handler prototypes.
395	* @{
396	*/
397	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmiNested;
398	static FNVMXEXITHANDLER hmR0VmxExitTripleFaultNested;
399	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindowNested;
400	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindowNested;
401	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitchNested;
402	static FNVMXEXITHANDLER hmR0VmxExitHltNested;
403	static FNVMXEXITHANDLER hmR0VmxExitInvlpgNested;
404	static FNVMXEXITHANDLER hmR0VmxExitRdpmcNested;
405	static FNVMXEXITHANDLER hmR0VmxExitVmreadVmwriteNested;
406	static FNVMXEXITHANDLER hmR0VmxExitRdtscNested;
407	static FNVMXEXITHANDLER hmR0VmxExitMovCRxNested;
408	static FNVMXEXITHANDLER hmR0VmxExitMovDRxNested;
409	static FNVMXEXITHANDLER hmR0VmxExitIoInstrNested;
410	static FNVMXEXITHANDLER hmR0VmxExitRdmsrNested;
411	static FNVMXEXITHANDLER hmR0VmxExitWrmsrNested;
412	static FNVMXEXITHANDLER hmR0VmxExitMwaitNested;
413	static FNVMXEXITHANDLER hmR0VmxExitMtfNested;
414	static FNVMXEXITHANDLER hmR0VmxExitMonitorNested;
415	static FNVMXEXITHANDLER hmR0VmxExitPauseNested;
416	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThresholdNested;
417	static FNVMXEXITHANDLER hmR0VmxExitApicAccessNested;
418	static FNVMXEXITHANDLER hmR0VmxExitApicWriteNested;
419	static FNVMXEXITHANDLER hmR0VmxExitVirtEoiNested;
420	static FNVMXEXITHANDLER hmR0VmxExitRdtscpNested;
421	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvdNested;
422	static FNVMXEXITHANDLER hmR0VmxExitInvpcidNested;
423	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestStateNested;
424	static FNVMXEXITHANDLER hmR0VmxExitInstrNested;
425	static FNVMXEXITHANDLER hmR0VmxExitInstrWithInfoNested;
426	/** @} */
427	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
428
429
430	/*********************************************************************************************************************************
431	* Global Variables *
432	*********************************************************************************************************************************/
433	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
434	/**
435	* Array of all VMCS fields.
436	* Any fields added to the VT-x spec. should be added here.
437	*
438	* Currently only used to derive shadow VMCS fields for hardware-assisted execution
439	* of nested-guests.
440	*/
441	static const uint32_t g_aVmcsFields[] =
442	{
443	/* 16-bit control fields. */
444	VMX_VMCS16_VPID,
445	VMX_VMCS16_POSTED_INT_NOTIFY_VECTOR,
446	VMX_VMCS16_EPTP_INDEX,
447
448	/* 16-bit guest-state fields. */
449	VMX_VMCS16_GUEST_ES_SEL,
450	VMX_VMCS16_GUEST_CS_SEL,
451	VMX_VMCS16_GUEST_SS_SEL,
452	VMX_VMCS16_GUEST_DS_SEL,
453	VMX_VMCS16_GUEST_FS_SEL,
454	VMX_VMCS16_GUEST_GS_SEL,
455	VMX_VMCS16_GUEST_LDTR_SEL,
456	VMX_VMCS16_GUEST_TR_SEL,
457	VMX_VMCS16_GUEST_INTR_STATUS,
458	VMX_VMCS16_GUEST_PML_INDEX,
459
460	/* 16-bits host-state fields. */
461	VMX_VMCS16_HOST_ES_SEL,
462	VMX_VMCS16_HOST_CS_SEL,
463	VMX_VMCS16_HOST_SS_SEL,
464	VMX_VMCS16_HOST_DS_SEL,
465	VMX_VMCS16_HOST_FS_SEL,
466	VMX_VMCS16_HOST_GS_SEL,
467	VMX_VMCS16_HOST_TR_SEL,
468
469	/* 64-bit control fields. */
470	VMX_VMCS64_CTRL_IO_BITMAP_A_FULL,
471	VMX_VMCS64_CTRL_IO_BITMAP_A_HIGH,
472	VMX_VMCS64_CTRL_IO_BITMAP_B_FULL,
473	VMX_VMCS64_CTRL_IO_BITMAP_B_HIGH,
474	VMX_VMCS64_CTRL_MSR_BITMAP_FULL,
475	VMX_VMCS64_CTRL_MSR_BITMAP_HIGH,
476	VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL,
477	VMX_VMCS64_CTRL_EXIT_MSR_STORE_HIGH,
478	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL,
479	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_HIGH,
480	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL,
481	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_HIGH,
482	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL,
483	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_HIGH,
484	VMX_VMCS64_CTRL_EXEC_PML_ADDR_FULL,
485	VMX_VMCS64_CTRL_EXEC_PML_ADDR_HIGH,
486	VMX_VMCS64_CTRL_TSC_OFFSET_FULL,
487	VMX_VMCS64_CTRL_TSC_OFFSET_HIGH,
488	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL,
489	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_HIGH,
490	VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL,
491	VMX_VMCS64_CTRL_APIC_ACCESSADDR_HIGH,
492	VMX_VMCS64_CTRL_POSTED_INTR_DESC_FULL,
493	VMX_VMCS64_CTRL_POSTED_INTR_DESC_HIGH,
494	VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL,
495	VMX_VMCS64_CTRL_VMFUNC_CTRLS_HIGH,
496	VMX_VMCS64_CTRL_EPTP_FULL,
497	VMX_VMCS64_CTRL_EPTP_HIGH,
498	VMX_VMCS64_CTRL_EOI_BITMAP_0_FULL,
499	VMX_VMCS64_CTRL_EOI_BITMAP_0_HIGH,
500	VMX_VMCS64_CTRL_EOI_BITMAP_1_FULL,
501	VMX_VMCS64_CTRL_EOI_BITMAP_1_HIGH,
502	VMX_VMCS64_CTRL_EOI_BITMAP_2_FULL,
503	VMX_VMCS64_CTRL_EOI_BITMAP_2_HIGH,
504	VMX_VMCS64_CTRL_EOI_BITMAP_3_FULL,
505	VMX_VMCS64_CTRL_EOI_BITMAP_3_HIGH,
506	VMX_VMCS64_CTRL_EPTP_LIST_FULL,
507	VMX_VMCS64_CTRL_EPTP_LIST_HIGH,
508	VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL,
509	VMX_VMCS64_CTRL_VMREAD_BITMAP_HIGH,
510	VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL,
511	VMX_VMCS64_CTRL_VMWRITE_BITMAP_HIGH,
512	VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_FULL,
513	VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_HIGH,
514	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_FULL,
515	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_HIGH,
516	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_FULL,
517	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_HIGH,
518	VMX_VMCS64_CTRL_TSC_MULTIPLIER_FULL,
519	VMX_VMCS64_CTRL_TSC_MULTIPLIER_HIGH,
520
521	/* 64-bit read-only data fields. */
522	VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL,
523	VMX_VMCS64_RO_GUEST_PHYS_ADDR_HIGH,
524
525	/* 64-bit guest-state fields. */
526	VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL,
527	VMX_VMCS64_GUEST_VMCS_LINK_PTR_HIGH,
528	VMX_VMCS64_GUEST_DEBUGCTL_FULL,
529	VMX_VMCS64_GUEST_DEBUGCTL_HIGH,
530	VMX_VMCS64_GUEST_PAT_FULL,
531	VMX_VMCS64_GUEST_PAT_HIGH,
532	VMX_VMCS64_GUEST_EFER_FULL,
533	VMX_VMCS64_GUEST_EFER_HIGH,
534	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL,
535	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_HIGH,
536	VMX_VMCS64_GUEST_PDPTE0_FULL,
537	VMX_VMCS64_GUEST_PDPTE0_HIGH,
538	VMX_VMCS64_GUEST_PDPTE1_FULL,
539	VMX_VMCS64_GUEST_PDPTE1_HIGH,
540	VMX_VMCS64_GUEST_PDPTE2_FULL,
541	VMX_VMCS64_GUEST_PDPTE2_HIGH,
542	VMX_VMCS64_GUEST_PDPTE3_FULL,
543	VMX_VMCS64_GUEST_PDPTE3_HIGH,
544	VMX_VMCS64_GUEST_BNDCFGS_FULL,
545	VMX_VMCS64_GUEST_BNDCFGS_HIGH,
546
547	/* 64-bit host-state fields. */
548	VMX_VMCS64_HOST_PAT_FULL,
549	VMX_VMCS64_HOST_PAT_HIGH,
550	VMX_VMCS64_HOST_EFER_FULL,
551	VMX_VMCS64_HOST_EFER_HIGH,
552	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL,
553	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_HIGH,
554
555	/* 32-bit control fields. */
556	VMX_VMCS32_CTRL_PIN_EXEC,
557	VMX_VMCS32_CTRL_PROC_EXEC,
558	VMX_VMCS32_CTRL_EXCEPTION_BITMAP,
559	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK,
560	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH,
561	VMX_VMCS32_CTRL_CR3_TARGET_COUNT,
562	VMX_VMCS32_CTRL_EXIT,
563	VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT,
564	VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT,
565	VMX_VMCS32_CTRL_ENTRY,
566	VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT,
567	VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO,
568	VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE,
569	VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH,
570	VMX_VMCS32_CTRL_TPR_THRESHOLD,
571	VMX_VMCS32_CTRL_PROC_EXEC2,
572	VMX_VMCS32_CTRL_PLE_GAP,
573	VMX_VMCS32_CTRL_PLE_WINDOW,
574
575	/* 32-bits read-only fields. */
576	VMX_VMCS32_RO_VM_INSTR_ERROR,
577	VMX_VMCS32_RO_EXIT_REASON,
578	VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO,
579	VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE,
580	VMX_VMCS32_RO_IDT_VECTORING_INFO,
581	VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE,
582	VMX_VMCS32_RO_EXIT_INSTR_LENGTH,
583	VMX_VMCS32_RO_EXIT_INSTR_INFO,
584
585	/* 32-bit guest-state fields. */
586	VMX_VMCS32_GUEST_ES_LIMIT,
587	VMX_VMCS32_GUEST_CS_LIMIT,
588	VMX_VMCS32_GUEST_SS_LIMIT,
589	VMX_VMCS32_GUEST_DS_LIMIT,
590	VMX_VMCS32_GUEST_FS_LIMIT,
591	VMX_VMCS32_GUEST_GS_LIMIT,
592	VMX_VMCS32_GUEST_LDTR_LIMIT,
593	VMX_VMCS32_GUEST_TR_LIMIT,
594	VMX_VMCS32_GUEST_GDTR_LIMIT,
595	VMX_VMCS32_GUEST_IDTR_LIMIT,
596	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
597	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
598	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
599	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
600	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
601	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS,
602	VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS,
603	VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS,
604	VMX_VMCS32_GUEST_INT_STATE,
605	VMX_VMCS32_GUEST_ACTIVITY_STATE,
606	VMX_VMCS32_GUEST_SMBASE,
607	VMX_VMCS32_GUEST_SYSENTER_CS,
608	VMX_VMCS32_PREEMPT_TIMER_VALUE,
609
610	/* 32-bit host-state fields. */
611	VMX_VMCS32_HOST_SYSENTER_CS,
612
613	/* Natural-width control fields. */
614	VMX_VMCS_CTRL_CR0_MASK,
615	VMX_VMCS_CTRL_CR4_MASK,
616	VMX_VMCS_CTRL_CR0_READ_SHADOW,
617	VMX_VMCS_CTRL_CR4_READ_SHADOW,
618	VMX_VMCS_CTRL_CR3_TARGET_VAL0,
619	VMX_VMCS_CTRL_CR3_TARGET_VAL1,
620	VMX_VMCS_CTRL_CR3_TARGET_VAL2,
621	VMX_VMCS_CTRL_CR3_TARGET_VAL3,
622
623	/* Natural-width read-only data fields. */
624	VMX_VMCS_RO_EXIT_QUALIFICATION,
625	VMX_VMCS_RO_IO_RCX,
626	VMX_VMCS_RO_IO_RSI,
627	VMX_VMCS_RO_IO_RDI,
628	VMX_VMCS_RO_IO_RIP,
629	VMX_VMCS_RO_GUEST_LINEAR_ADDR,
630
631	/* Natural-width guest-state field */
632	VMX_VMCS_GUEST_CR0,
633	VMX_VMCS_GUEST_CR3,
634	VMX_VMCS_GUEST_CR4,
635	VMX_VMCS_GUEST_ES_BASE,
636	VMX_VMCS_GUEST_CS_BASE,
637	VMX_VMCS_GUEST_SS_BASE,
638	VMX_VMCS_GUEST_DS_BASE,
639	VMX_VMCS_GUEST_FS_BASE,
640	VMX_VMCS_GUEST_GS_BASE,
641	VMX_VMCS_GUEST_LDTR_BASE,
642	VMX_VMCS_GUEST_TR_BASE,
643	VMX_VMCS_GUEST_GDTR_BASE,
644	VMX_VMCS_GUEST_IDTR_BASE,
645	VMX_VMCS_GUEST_DR7,
646	VMX_VMCS_GUEST_RSP,
647	VMX_VMCS_GUEST_RIP,
648	VMX_VMCS_GUEST_RFLAGS,
649	VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,
650	VMX_VMCS_GUEST_SYSENTER_ESP,
651	VMX_VMCS_GUEST_SYSENTER_EIP,
652
653	/* Natural-width host-state fields */
654	VMX_VMCS_HOST_CR0,
655	VMX_VMCS_HOST_CR3,
656	VMX_VMCS_HOST_CR4,
657	VMX_VMCS_HOST_FS_BASE,
658	VMX_VMCS_HOST_GS_BASE,
659	VMX_VMCS_HOST_TR_BASE,
660	VMX_VMCS_HOST_GDTR_BASE,
661	VMX_VMCS_HOST_IDTR_BASE,
662	VMX_VMCS_HOST_SYSENTER_ESP,
663	VMX_VMCS_HOST_SYSENTER_EIP,
664	VMX_VMCS_HOST_RSP,
665	VMX_VMCS_HOST_RIP
666	};
667	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
668
669	static const uint32_t g_aVmcsSegBase[] =
670	{
671	VMX_VMCS_GUEST_ES_BASE,
672	VMX_VMCS_GUEST_CS_BASE,
673	VMX_VMCS_GUEST_SS_BASE,
674	VMX_VMCS_GUEST_DS_BASE,
675	VMX_VMCS_GUEST_FS_BASE,
676	VMX_VMCS_GUEST_GS_BASE
677	};
678	static const uint32_t g_aVmcsSegSel[] =
679	{
680	VMX_VMCS16_GUEST_ES_SEL,
681	VMX_VMCS16_GUEST_CS_SEL,
682	VMX_VMCS16_GUEST_SS_SEL,
683	VMX_VMCS16_GUEST_DS_SEL,
684	VMX_VMCS16_GUEST_FS_SEL,
685	VMX_VMCS16_GUEST_GS_SEL
686	};
687	static const uint32_t g_aVmcsSegLimit[] =
688	{
689	VMX_VMCS32_GUEST_ES_LIMIT,
690	VMX_VMCS32_GUEST_CS_LIMIT,
691	VMX_VMCS32_GUEST_SS_LIMIT,
692	VMX_VMCS32_GUEST_DS_LIMIT,
693	VMX_VMCS32_GUEST_FS_LIMIT,
694	VMX_VMCS32_GUEST_GS_LIMIT
695	};
696	static const uint32_t g_aVmcsSegAttr[] =
697	{
698	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
699	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
700	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
701	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
702	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
703	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
704	};
705	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
706	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
707	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
708	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
709
710	#ifdef HMVMX_USE_FUNCTION_TABLE
711	/**
712	* VMX_EXIT dispatch table.
713	*/
714	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
715	{
716	/* 0 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
717	/* 1 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
718	/* 2 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
719	/* 3 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitErrUnexpected,
720	/* 4 VMX_EXIT_SIPI */ hmR0VmxExitErrUnexpected,
721	/* 5 VMX_EXIT_IO_SMI */ hmR0VmxExitErrUnexpected,
722	/* 6 VMX_EXIT_SMI */ hmR0VmxExitErrUnexpected,
723	/* 7 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
724	/* 8 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
725	/* 9 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
726	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
727	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
728	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
729	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
730	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
731	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
732	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
733	/* 17 VMX_EXIT_RSM */ hmR0VmxExitErrUnexpected,
734	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
735	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
736	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
737	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
738	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
739	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
740	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
741	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
742	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
743	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
744	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
745	#else
746	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
747	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
748	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
749	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
750	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
751	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
752	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
753	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
754	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
755	#endif
756	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
757	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
758	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
759	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
760	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
761	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
762	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrUnexpected,
763	/* 35 UNDEFINED */ hmR0VmxExitErrUnexpected,
764	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
765	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
766	/* 38 UNDEFINED */ hmR0VmxExitErrUnexpected,
767	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
768	/* 40 VMX_EXIT_PAUSE */ hmR0VmxExitPause,
769	/* 41 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUnexpected,
770	/* 42 UNDEFINED */ hmR0VmxExitErrUnexpected,
771	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
772	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
773	/* 45 VMX_EXIT_VIRTUALIZED_EOI */ hmR0VmxExitErrUnexpected,
774	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitErrUnexpected,
775	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitErrUnexpected,
776	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
777	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
778	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
779	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
780	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
781	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
782	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInvvpid,
783	#else
784	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
785	#endif
786	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
787	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
788	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUnexpected,
789	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitErrUnexpected,
790	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
791	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitErrUnexpected,
792	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUnexpected,
793	/* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUnexpected,
794	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUnexpected,
795	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitErrUnexpected,
796	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitErrUnexpected,
797	/* 65 UNDEFINED */ hmR0VmxExitErrUnexpected,
798	/* 66 VMX_EXIT_SPP_EVENT */ hmR0VmxExitErrUnexpected,
799	/* 67 VMX_EXIT_UMWAIT */ hmR0VmxExitErrUnexpected,
800	/* 68 VMX_EXIT_TPAUSE */ hmR0VmxExitErrUnexpected,
801	};
802	#endif /* HMVMX_USE_FUNCTION_TABLE */
803
804	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
805	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
806	{
807	/* 0 */ "(Not Used)",
808	/* 1 */ "VMCALL executed in VMX root operation.",
809	/* 2 */ "VMCLEAR with invalid physical address.",
810	/* 3 */ "VMCLEAR with VMXON pointer.",
811	/* 4 */ "VMLAUNCH with non-clear VMCS.",
812	/* 5 */ "VMRESUME with non-launched VMCS.",
813	/* 6 */ "VMRESUME after VMXOFF",
814	/* 7 */ "VM-entry with invalid control fields.",
815	/* 8 */ "VM-entry with invalid host state fields.",
816	/* 9 */ "VMPTRLD with invalid physical address.",
817	/* 10 */ "VMPTRLD with VMXON pointer.",
818	/* 11 */ "VMPTRLD with incorrect revision identifier.",
819	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
820	/* 13 */ "VMWRITE to read-only VMCS component.",
821	/* 14 */ "(Not Used)",
822	/* 15 */ "VMXON executed in VMX root operation.",
823	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
824	/* 17 */ "VM-entry with non-launched executing VMCS.",
825	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
826	/* 19 */ "VMCALL with non-clear VMCS.",
827	/* 20 */ "VMCALL with invalid VM-exit control fields.",
828	/* 21 */ "(Not Used)",
829	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
830	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
831	/* 24 */ "VMCALL with invalid SMM-monitor features.",
832	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
833	/* 26 */ "VM-entry with events blocked by MOV SS.",
834	/* 27 */ "(Not Used)",
835	/* 28 */ "Invalid operand to INVEPT/INVVPID."
836	};
837	#endif /* VBOX_STRICT && LOG_ENABLED */
838
839
840	/**
841	* Get the CR0 guest/host mask that does not change through the lifetime of a VM.
842	*
843	* Any bit set in this mask is owned by the host/hypervisor and would cause a
844	* VM-exit when modified by the guest.
845	*
846	* @returns The static CR0 guest/host mask.
847	* @param pVCpu The cross context virtual CPU structure.
848	*/
849	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr0Mask(PCVMCPU pVCpu)
850	{
851	/*
852	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
853	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
854	*/
855	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
856	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
857	* and @bugref{6944}. */
858	PVM pVM = pVCpu->CTX_SUFF(pVM);
859	return ( X86_CR0_PE
860	\| X86_CR0_NE
861	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
862	\| X86_CR0_PG
863	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
864	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
865	\| X86_CR0_NW); /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
866	}
867
868
869	/**
870	* Gets the CR4 guest/host mask that does not change through the lifetime of a VM.
871	*
872	* Any bit set in this mask is owned by the host/hypervisor and would cause a
873	* VM-exit when modified by the guest.
874	*
875	* @returns The static CR4 guest/host mask.
876	* @param pVCpu The cross context virtual CPU structure.
877	*/
878	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr4Mask(PCVMCPU pVCpu)
879	{
880	/*
881	* We need to look at the host features here (for e.g. OSXSAVE, PCID) because
882	* these bits are reserved on hardware that does not support them. Since the
883	* CPU cannot refer to our virtual CPUID, we need to intercept CR4 changes to
884	* these bits and handle it depending on whether we expose them to the guest.
885	*/
886	PVM pVM = pVCpu->CTX_SUFF(pVM);
887	bool const fXSaveRstor = pVM->cpum.ro.HostFeatures.fXSaveRstor;
888	bool const fPcid = pVM->cpum.ro.HostFeatures.fPcid;
889	return ( X86_CR4_VMXE
890	\| X86_CR4_VME
891	\| X86_CR4_PAE
892	\| X86_CR4_PGE
893	\| X86_CR4_PSE
894	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
895	\| (fPcid ? X86_CR4_PCIDE : 0));
896	}
897
898
899	/**
900	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
901	* area.
902	*
903	* @returns @c true if it's different, @c false otherwise.
904	* @param pVmcsInfo The VMCS info. object.
905	*/
906	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
907	{
908	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
909	&& pVmcsInfo->pvGuestMsrStore);
910	}
911
912
913	/**
914	* Sets the given Processor-based VM-execution controls.
915	*
916	* @param pVmxTransient The VMX-transient structure.
917	* @param uProcCtls The Processor-based VM-execution controls to set.
918	*/
919	static void hmR0VmxSetProcCtlsVmcs(PVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
920	{
921	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
922	if ((pVmcsInfo->u32ProcCtls & uProcCtls) != uProcCtls)
923	{
924	pVmcsInfo->u32ProcCtls \|= uProcCtls;
925	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
926	AssertRC(rc);
927	}
928	}
929
930
931	/**
932	* Removes the given Processor-based VM-execution controls.
933	*
934	* @param pVCpu The cross context virtual CPU structure.
935	* @param pVmxTransient The VMX-transient structure.
936	* @param uProcCtls The Processor-based VM-execution controls to remove.
937	*
938	* @remarks When executing a nested-guest, this will not remove any of the specified
939	* controls if the guest hypervisor has set any one of them.
940	*/
941	static void hmR0VmxRemoveProcCtlsVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
942	{
943	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
944	if (pVmcsInfo->u32ProcCtls & uProcCtls)
945	{
946	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
947	bool const fRemoveCtls = !pVmxTransient->fIsNestedGuest
948	? true
949	: !CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uProcCtls);
950	#else
951	NOREF(pVCpu);
952	bool const fRemoveCtls = true;
953	#endif
954	if (fRemoveCtls)
955	{
956	pVmcsInfo->u32ProcCtls &= ~uProcCtls;
957	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
958	AssertRC(rc);
959	}
960	}
961	}
962
963
964	/**
965	* Sets the TSC offset for the current VMCS.
966	*
967	* @param pVCpu The cross context virtual CPU structure.
968	* @param uTscOffset The TSC offset to set.
969	* @param pVmcsInfo The VMCS info. object.
970	*/
971	static void hmR0VmxSetTscOffsetVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t uTscOffset)
972	{
973	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
974	if (pVmcsInfo->u64TscOffset != uTscOffset)
975	{
976	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
977	AssertRC(rc);
978	pVmcsInfo->u64TscOffset = uTscOffset;
979	}
980	}
981
982
983	/**
984	* Adds one or more exceptions to the exception bitmap and commits it to the current
985	* VMCS.
986	*
987	* @returns VBox status code.
988	* @param pVmxTransient The VMX-transient structure.
989	* @param uXcptMask The exception(s) to add.
990	*/
991	static int hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
992	{
993	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
994	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
995	if ((uXcptBitmap & uXcptMask) != uXcptMask)
996	{
997	uXcptBitmap \|= uXcptMask;
998	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
999	AssertRCReturn(rc, rc);
1000	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
1001	}
1002	return VINF_SUCCESS;
1003	}
1004
1005
1006	/**
1007	* Adds an exception to the exception bitmap and commits it to the current VMCS.
1008	*
1009	* @returns VBox status code.
1010	* @param pVmxTransient The VMX-transient structure.
1011	* @param uXcpt The exception to add.
1012	*/
1013	static int hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1014	{
1015	Assert(uXcpt <= X86_XCPT_LAST);
1016	return hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
1017	}
1018
1019
1020	/**
1021	* Remove one or more exceptions from the exception bitmap and commits it to the
1022	* current VMCS.
1023	*
1024	* This takes care of not removing the exception intercept if a nested-guest
1025	* requires the exception to be intercepted.
1026	*
1027	* @returns VBox status code.
1028	* @param pVCpu The cross context virtual CPU structure.
1029	* @param pVmxTransient The VMX-transient structure.
1030	* @param uXcptMask The exception(s) to remove.
1031	*/
1032	static int hmR0VmxRemoveXcptInterceptMask(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
1033	{
1034	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1035	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
1036	if (u32XcptBitmap & uXcptMask)
1037	{
1038	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1039	if (!pVmxTransient->fIsNestedGuest)
1040	{ /* likely */ }
1041	else
1042	{
1043	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
1044	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
1045	}
1046	#endif
1047	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
1048	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
1049	\| RT_BIT(X86_XCPT_DE)
1050	\| RT_BIT(X86_XCPT_NM)
1051	\| RT_BIT(X86_XCPT_TS)
1052	\| RT_BIT(X86_XCPT_UD)
1053	\| RT_BIT(X86_XCPT_NP)
1054	\| RT_BIT(X86_XCPT_SS)
1055	\| RT_BIT(X86_XCPT_GP)
1056	\| RT_BIT(X86_XCPT_PF)
1057	\| RT_BIT(X86_XCPT_MF));
1058	#elif defined(HMVMX_ALWAYS_TRAP_PF)
1059	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
1060	#endif
1061	if (uXcptMask)
1062	{
1063	/* Validate we are not removing any essential exception intercepts. */
1064	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF))); RT_NOREF(pVCpu);
1065	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
1066	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
1067
1068	/* Remove it from the exception bitmap. */
1069	u32XcptBitmap &= ~uXcptMask;
1070
1071	/* Commit and update the cache if necessary. */
1072	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
1073	{
1074	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1075	AssertRCReturn(rc, rc);
1076	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
1077	}
1078	}
1079	}
1080	return VINF_SUCCESS;
1081	}
1082
1083
1084	/**
1085	* Remove an exceptions from the exception bitmap and commits it to the current
1086	* VMCS.
1087	*
1088	* @returns VBox status code.
1089	* @param pVCpu The cross context virtual CPU structure.
1090	* @param pVmxTransient The VMX-transient structure.
1091	* @param uXcpt The exception to remove.
1092	*/
1093	static int hmR0VmxRemoveXcptIntercept(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1094	{
1095	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
1096	}
1097
1098
1099	/**
1100	* Loads the VMCS specified by the VMCS info. object.
1101	*
1102	* @returns VBox status code.
1103	* @param pVmcsInfo The VMCS info. object.
1104	*
1105	* @remarks Can be called with interrupts disabled.
1106	*/
1107	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
1108	{
1109	Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
1110	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1111
1112	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
1113	if (RT_SUCCESS(rc))
1114	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1115	return rc;
1116	}
1117
1118
1119	/**
1120	* Clears the VMCS specified by the VMCS info. object.
1121	*
1122	* @returns VBox status code.
1123	* @param pVmcsInfo The VMCS info. object.
1124	*
1125	* @remarks Can be called with interrupts disabled.
1126	*/
1127	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
1128	{
1129	Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
1130	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1131
1132	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
1133	if (RT_SUCCESS(rc))
1134	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1135	return rc;
1136	}
1137
1138
1139	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1140	/**
1141	* Loads the shadow VMCS specified by the VMCS info. object.
1142	*
1143	* @returns VBox status code.
1144	* @param pVmcsInfo The VMCS info. object.
1145	*
1146	* @remarks Can be called with interrupts disabled.
1147	*/
1148	static int hmR0VmxLoadShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1149	{
1150	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1151	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1152
1153	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysShadowVmcs);
1154	if (RT_SUCCESS(rc))
1155	pVmcsInfo->fShadowVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1156	return rc;
1157	}
1158
1159
1160	/**
1161	* Clears the shadow VMCS specified by the VMCS info. object.
1162	*
1163	* @returns VBox status code.
1164	* @param pVmcsInfo The VMCS info. object.
1165	*
1166	* @remarks Can be called with interrupts disabled.
1167	*/
1168	static int hmR0VmxClearShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1169	{
1170	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1171	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1172
1173	int rc = VMXClearVmcs(pVmcsInfo->HCPhysShadowVmcs);
1174	if (RT_SUCCESS(rc))
1175	pVmcsInfo->fShadowVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1176	return rc;
1177	}
1178
1179
1180	/**
1181	* Switches from and to the specified VMCSes.
1182	*
1183	* @returns VBox status code.
1184	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
1185	* @param pVmcsInfoTo The VMCS info. object we are switching to.
1186	*
1187	* @remarks Called with interrupts disabled.
1188	*/
1189	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
1190	{
1191	/*
1192	* Clear the VMCS we are switching out if it has not already been cleared.
1193	* This will sync any CPU internal data back to the VMCS.
1194	*/
1195	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1196	{
1197	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
1198	if (RT_SUCCESS(rc))
1199	{
1200	/*
1201	* The shadow VMCS, if any, would not be active at this point since we
1202	* would have cleared it while importing the virtual hardware-virtualization
1203	* state as part the VMLAUNCH/VMRESUME VM-exit. Hence, there's no need to
1204	* clear the shadow VMCS here, just assert for safety.
1205	*/
1206	Assert(!pVmcsInfoFrom->pvShadowVmcs \|\| pVmcsInfoFrom->fShadowVmcsState == VMX_V_VMCS_LAUNCH_STATE_CLEAR);
1207	}
1208	else
1209	return rc;
1210	}
1211
1212	/*
1213	* Clear the VMCS we are switching to if it has not already been cleared.
1214	* This will initialize the VMCS launch state to "clear" required for loading it.
1215	*
1216	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
1217	*/
1218	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1219	{
1220	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
1221	if (RT_SUCCESS(rc))
1222	{ /* likely */ }
1223	else
1224	return rc;
1225	}
1226
1227	/*
1228	* Finally, load the VMCS we are switching to.
1229	*/
1230	return hmR0VmxLoadVmcs(pVmcsInfoTo);
1231	}
1232
1233
1234	/**
1235	* Switches between the guest VMCS and the nested-guest VMCS as specified by the
1236	* caller.
1237	*
1238	* @returns VBox status code.
1239	* @param pVCpu The cross context virtual CPU structure.
1240	* @param fSwitchToNstGstVmcs Whether to switch to the nested-guest VMCS (pass
1241	* true) or guest VMCS (pass false).
1242	*/
1243	static int hmR0VmxSwitchToGstOrNstGstVmcs(PVMCPU pVCpu, bool fSwitchToNstGstVmcs)
1244	{
1245	/* Ensure we have synced everything from the guest-CPU context to the VMCS before switching. */
1246	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
1247
1248	PVMXVMCSINFO pVmcsInfoFrom;
1249	PVMXVMCSINFO pVmcsInfoTo;
1250	if (fSwitchToNstGstVmcs)
1251	{
1252	pVmcsInfoFrom = &pVCpu->hm.s.vmx.VmcsInfo;
1253	pVmcsInfoTo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1254	}
1255	else
1256	{
1257	pVmcsInfoFrom = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1258	pVmcsInfoTo = &pVCpu->hm.s.vmx.VmcsInfo;
1259	}
1260
1261	/*
1262	* Disable interrupts to prevent being preempted while we switch the current VMCS as the
1263	* preemption hook code path acquires the current VMCS.
1264	*/
1265	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1266
1267	int rc = hmR0VmxSwitchVmcs(pVmcsInfoFrom, pVmcsInfoTo);
1268	if (RT_SUCCESS(rc))
1269	{
1270	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fSwitchToNstGstVmcs;
1271
1272	/*
1273	* If we are switching to a VMCS that was executed on a different host CPU or was
1274	* never executed before, flag that we need to export the host state before executing
1275	* guest/nested-guest code using hardware-assisted VMX.
1276	*
1277	* This could probably be done in a preemptible context since the preemption hook
1278	* will flag the necessary change in host context. However, since preemption is
1279	* already disabled and to avoid making assumptions about host specific code in
1280	* RTMpCpuId when called with preemption enabled, we'll do this while preemption is
1281	* disabled.
1282	*/
1283	if (pVmcsInfoTo->idHostCpu == RTMpCpuId())
1284	{ /* likely */ }
1285	else
1286	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE);
1287
1288	ASMSetFlags(fEFlags);
1289
1290	/*
1291	* We use a different VM-exit MSR-store areas for the guest and nested-guest. Hence,
1292	* flag that we need to update the host MSR values there. Even if we decide in the
1293	* future to share the VM-exit MSR-store area page between the guest and nested-guest,
1294	* if its content differs, we would have to update the host MSRs anyway.
1295	*/
1296	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
1297	}
1298	else
1299	ASMSetFlags(fEFlags);
1300	return rc;
1301	}
1302	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
1303
1304
1305	/**
1306	* Updates the VM's last error record.
1307	*
1308	* If there was a VMX instruction error, reads the error data from the VMCS and
1309	* updates VCPU's last error record as well.
1310	*
1311	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1312	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
1313	* VERR_VMX_INVALID_VMCS_FIELD.
1314	* @param rc The error code.
1315	*/
1316	static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
1317	{
1318	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
1319	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
1320	{
1321	AssertPtrReturnVoid(pVCpu);
1322	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
1323	}
1324	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
1325	}
1326
1327
1328	#ifdef VBOX_STRICT
1329	/**
1330	* Reads the VM-entry interruption-information field from the VMCS into the VMX
1331	* transient structure.
1332	*
1333	* @returns VBox status code.
1334	* @param pVmxTransient The VMX-transient structure.
1335	*
1336	* @remarks No-long-jump zone!!!
1337	*/
1338	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1339	{
1340	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
1341	AssertRCReturn(rc, rc);
1342	return VINF_SUCCESS;
1343	}
1344
1345
1346	/**
1347	* Reads the VM-entry exception error code field from the VMCS into
1348	* the VMX transient structure.
1349	*
1350	* @returns VBox status code.
1351	* @param pVmxTransient The VMX-transient structure.
1352	*
1353	* @remarks No-long-jump zone!!!
1354	*/
1355	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1356	{
1357	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
1358	AssertRCReturn(rc, rc);
1359	return VINF_SUCCESS;
1360	}
1361
1362
1363	/**
1364	* Reads the VM-entry exception error code field from the VMCS into
1365	* the VMX transient structure.
1366	*
1367	* @returns VBox status code.
1368	* @param pVmxTransient The VMX-transient structure.
1369	*
1370	* @remarks No-long-jump zone!!!
1371	*/
1372	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1373	{
1374	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
1375	AssertRCReturn(rc, rc);
1376	return VINF_SUCCESS;
1377	}
1378	#endif /* VBOX_STRICT */
1379
1380
1381	/**
1382	* Reads the VM-exit interruption-information field from the VMCS into the VMX
1383	* transient structure.
1384	*
1385	* @returns VBox status code.
1386	* @param pVmxTransient The VMX-transient structure.
1387	*/
1388	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1389	{
1390	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
1391	{
1392	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1393	AssertRCReturn(rc,rc);
1394	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
1395	}
1396	return VINF_SUCCESS;
1397	}
1398
1399
1400	/**
1401	* Reads the VM-exit interruption error code from the VMCS into the VMX
1402	* transient structure.
1403	*
1404	* @returns VBox status code.
1405	* @param pVmxTransient The VMX-transient structure.
1406	*/
1407	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1408	{
1409	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
1410	{
1411	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1412	AssertRCReturn(rc, rc);
1413	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1414	}
1415	return VINF_SUCCESS;
1416	}
1417
1418
1419	/**
1420	* Reads the VM-exit instruction length field from the VMCS into the VMX
1421	* transient structure.
1422	*
1423	* @returns VBox status code.
1424	* @param pVmxTransient The VMX-transient structure.
1425	*/
1426	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1427	{
1428	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1429	{
1430	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
1431	AssertRCReturn(rc, rc);
1432	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1433	}
1434	return VINF_SUCCESS;
1435	}
1436
1437
1438	/**
1439	* Reads the VM-exit instruction-information field from the VMCS into
1440	* the VMX transient structure.
1441	*
1442	* @returns VBox status code.
1443	* @param pVmxTransient The VMX-transient structure.
1444	*/
1445	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1446	{
1447	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1448	{
1449	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1450	AssertRCReturn(rc, rc);
1451	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1452	}
1453	return VINF_SUCCESS;
1454	}
1455
1456
1457	/**
1458	* Reads the Exit Qualification from the VMCS into the VMX transient structure.
1459	*
1460	* @returns VBox status code.
1461	* @param pVCpu The cross context virtual CPU structure of the
1462	* calling EMT. (Required for the VMCS cache case.)
1463	* @param pVmxTransient The VMX-transient structure.
1464	*/
1465	DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1466	{
1467	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1468	{
1469	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
1470	AssertRCReturn(rc, rc);
1471	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1472	}
1473	return VINF_SUCCESS;
1474	}
1475
1476
1477	/**
1478	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1479	*
1480	* @returns VBox status code.
1481	* @param pVCpu The cross context virtual CPU structure of the
1482	* calling EMT. (Required for the VMCS cache case.)
1483	* @param pVmxTransient The VMX-transient structure.
1484	*/
1485	DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1486	{
1487	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1488	{
1489	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); NOREF(pVCpu);
1490	AssertRCReturn(rc, rc);
1491	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1492	}
1493	return VINF_SUCCESS;
1494	}
1495
1496
1497	/**
1498	* Reads the Guest-physical address from the VMCS into the VMX transient structure.
1499	*
1500	* @returns VBox status code.
1501	* @param pVCpu The cross context virtual CPU structure of the
1502	* calling EMT. (Required for the VMCS cache case.)
1503	* @param pVmxTransient The VMX-transient structure.
1504	*/
1505	DECLINLINE(int) hmR0VmxReadGuestPhysicalAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1506	{
1507	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_PHYSICAL_ADDR))
1508	{
1509	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr); NOREF(pVCpu);
1510	AssertRCReturn(rc, rc);
1511	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_PHYSICAL_ADDR;
1512	}
1513	return VINF_SUCCESS;
1514	}
1515
1516
1517	/**
1518	* Reads the IDT-vectoring information field from the VMCS into the VMX
1519	* transient structure.
1520	*
1521	* @returns VBox status code.
1522	* @param pVmxTransient The VMX-transient structure.
1523	*
1524	* @remarks No-long-jump zone!!!
1525	*/
1526	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1527	{
1528	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1529	{
1530	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1531	AssertRCReturn(rc, rc);
1532	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1533	}
1534	return VINF_SUCCESS;
1535	}
1536
1537
1538	/**
1539	* Reads the IDT-vectoring error code from the VMCS into the VMX
1540	* transient structure.
1541	*
1542	* @returns VBox status code.
1543	* @param pVmxTransient The VMX-transient structure.
1544	*/
1545	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1546	{
1547	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1548	{
1549	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1550	AssertRCReturn(rc, rc);
1551	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1552	}
1553	return VINF_SUCCESS;
1554	}
1555
1556	#ifdef HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
1557	/**
1558	* Reads all relevant read-only VMCS fields into the VMX transient structure.
1559	*
1560	* @returns VBox status code.
1561	* @param pVCpu The cross context virtual CPU structure of the
1562	* calling EMT. (Required for the VMCS cache case.)
1563	* @param pVmxTransient The VMX-transient structure.
1564	*/
1565	static int hmR0VmxReadAllRoFieldsVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1566	{
1567	NOREF(pVCpu); /* Used implicitly by VMXReadVmcsGstN on 32-bit hosts. */
1568	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual);
1569	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
1570	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1571	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1572	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1573	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1574	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1575	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr);
1576	rc \|= VMXReadVmcsGstN(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr);
1577	AssertRCReturn(rc, rc);
1578	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION
1579	\| HMVMX_READ_EXIT_INSTR_LEN
1580	\| HMVMX_READ_EXIT_INSTR_INFO
1581	\| HMVMX_READ_IDT_VECTORING_INFO
1582	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE
1583	\| HMVMX_READ_EXIT_INTERRUPTION_INFO
1584	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
1585	\| HMVMX_READ_GUEST_LINEAR_ADDR
1586	\| HMVMX_READ_GUEST_PHYSICAL_ADDR;
1587	return VINF_SUCCESS;
1588	}
1589	#endif
1590
1591	/**
1592	* Enters VMX root mode operation on the current CPU.
1593	*
1594	* @returns VBox status code.
1595	* @param pVM The cross context VM structure. Can be
1596	* NULL, after a resume.
1597	* @param HCPhysCpuPage Physical address of the VMXON region.
1598	* @param pvCpuPage Pointer to the VMXON region.
1599	*/
1600	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1601	{
1602	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1603	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1604	Assert(pvCpuPage);
1605	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1606
1607	if (pVM)
1608	{
1609	/* Write the VMCS revision identifier to the VMXON region. */
1610	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1611	}
1612
1613	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1614	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1615
1616	/* Enable the VMX bit in CR4 if necessary. */
1617	RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1618
1619	/* Enter VMX root mode. */
1620	int rc = VMXEnable(HCPhysCpuPage);
1621	if (RT_FAILURE(rc))
1622	{
1623	if (!(uOldCr4 & X86_CR4_VMXE))
1624	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1625
1626	if (pVM)
1627	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1628	}
1629
1630	/* Restore interrupts. */
1631	ASMSetFlags(fEFlags);
1632	return rc;
1633	}
1634
1635
1636	/**
1637	* Exits VMX root mode operation on the current CPU.
1638	*
1639	* @returns VBox status code.
1640	*/
1641	static int hmR0VmxLeaveRootMode(void)
1642	{
1643	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1644
1645	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1646	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1647
1648	/* If we're for some reason not in VMX root mode, then don't leave it. */
1649	RTCCUINTREG const uHostCr4 = ASMGetCR4();
1650
1651	int rc;
1652	if (uHostCr4 & X86_CR4_VMXE)
1653	{
1654	/* Exit VMX root mode and clear the VMX bit in CR4. */
1655	VMXDisable();
1656	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1657	rc = VINF_SUCCESS;
1658	}
1659	else
1660	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1661
1662	/* Restore interrupts. */
1663	ASMSetFlags(fEFlags);
1664	return rc;
1665	}
1666
1667
1668	/**
1669	* Allocates and maps a physically contiguous page. The allocated page is
1670	* zero'd out (used by various VT-x structures).
1671	*
1672	* @returns IPRT status code.
1673	* @param pMemObj Pointer to the ring-0 memory object.
1674	* @param ppVirt Where to store the virtual address of the allocation.
1675	* @param pHCPhys Where to store the physical address of the allocation.
1676	*/
1677	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1678	{
1679	AssertPtr(pMemObj);
1680	AssertPtr(ppVirt);
1681	AssertPtr(pHCPhys);
1682	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1683	if (RT_FAILURE(rc))
1684	return rc;
1685	ppVirt = RTR0MemObjAddress(pMemObj);
1686	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1687	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1688	return VINF_SUCCESS;
1689	}
1690
1691
1692	/**
1693	* Frees and unmaps an allocated, physical page.
1694	*
1695	* @param pMemObj Pointer to the ring-0 memory object.
1696	* @param ppVirt Where to re-initialize the virtual address of allocation as
1697	* 0.
1698	* @param pHCPhys Where to re-initialize the physical address of the
1699	* allocation as 0.
1700	*/
1701	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1702	{
1703	AssertPtr(pMemObj);
1704	AssertPtr(ppVirt);
1705	AssertPtr(pHCPhys);
1706	/* NULL is valid, accepted and ignored by the free function below. */
1707	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1708	*pMemObj = NIL_RTR0MEMOBJ;
1709	*ppVirt = NULL;
1710	*pHCPhys = NIL_RTHCPHYS;
1711	}
1712
1713
1714	/**
1715	* Initializes a VMCS info. object.
1716	*
1717	* @param pVmcsInfo The VMCS info. object.
1718	*/
1719	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1720	{
1721	memset(pVmcsInfo, 0, sizeof(*pVmcsInfo));
1722
1723	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1724	Assert(pVmcsInfo->hMemObjShadowVmcs == NIL_RTR0MEMOBJ);
1725	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1726	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1727	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1728	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1729	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1730	pVmcsInfo->HCPhysShadowVmcs = NIL_RTHCPHYS;
1731	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1732	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1733	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1734	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1735	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1736	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1737	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1738	pVmcsInfo->idHostCpu = NIL_RTCPUID;
1739	}
1740
1741
1742	/**
1743	* Frees the VT-x structures for a VMCS info. object.
1744	*
1745	* @param pVM The cross context VM structure.
1746	* @param pVmcsInfo The VMCS info. object.
1747	*/
1748	static void hmR0VmxFreeVmcsInfo(PVM pVM, PVMXVMCSINFO pVmcsInfo)
1749	{
1750	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1751
1752	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1753	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1754	hmR0VmxPageFree(&pVmcsInfo->hMemObjShadowVmcs, &pVmcsInfo->pvShadowVmcs, &pVmcsInfo->HCPhysShadowVmcs);
1755	#endif
1756
1757	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1758	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1759
1760	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1761	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1762	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1763
1764	hmR0VmxInitVmcsInfo(pVmcsInfo);
1765	}
1766
1767
1768	/**
1769	* Allocates the VT-x structures for a VMCS info. object.
1770	*
1771	* @returns VBox status code.
1772	* @param pVCpu The cross context virtual CPU structure.
1773	* @param pVmcsInfo The VMCS info. object.
1774	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1775	*/
1776	static int hmR0VmxAllocVmcsInfo(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1777	{
1778	PVM pVM = pVCpu->CTX_SUFF(pVM);
1779
1780	/* Allocate the guest VM control structure (VMCS). */
1781	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1782	if (RT_SUCCESS(rc))
1783	{
1784	if (!fIsNstGstVmcs)
1785	{
1786	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1787	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1788	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjShadowVmcs, &pVmcsInfo->pvShadowVmcs, &pVmcsInfo->HCPhysShadowVmcs);
1789	#endif
1790	if (RT_SUCCESS(rc))
1791	{
1792	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1793	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1794	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1795	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic, NULL /pR3Ptr/);
1796	}
1797	}
1798	else
1799	{
1800	/* We don't yet support exposing VMCS shadowing to the guest. */
1801	Assert(pVmcsInfo->HCPhysShadowVmcs == NIL_RTHCPHYS);
1802	Assert(!pVmcsInfo->pvShadowVmcs);
1803
1804	/* Get the allocated virtual-APIC page from CPUM. */
1805	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
1806	{
1807	/** @todo NSTVMX: Get rid of this. There is no need to allocate a separate HC
1808	* page for this. Use the one provided by the nested-guest directly. */
1809	pVmcsInfo->pbVirtApic = (uint8_t *)CPUMGetGuestVmxVirtApicPage(pVCpu, &pVCpu->cpum.GstCtx,
1810	&pVmcsInfo->HCPhysVirtApic);
1811	Assert(pVmcsInfo->pbVirtApic);
1812	Assert(pVmcsInfo->HCPhysVirtApic && pVmcsInfo->HCPhysVirtApic != NIL_RTHCPHYS);
1813	}
1814	}
1815
1816	if (RT_SUCCESS(rc))
1817	{
1818	/*
1819	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1820	* transparent accesses of specific MSRs.
1821	*
1822	* If the condition for enabling MSR bitmaps changes here, don't forget to
1823	* update HMIsMsrBitmapActive().
1824	*
1825	* We don't share MSR bitmaps between the guest and nested-guest as we then
1826	* don't need to care about carefully restoring the guest MSR bitmap.
1827	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1828	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1829	* We also don't need to re-initialize the nested-guest MSR bitmap here as
1830	* we do that later while merging VMCS.
1831	*/
1832	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1833	{
1834	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1835	if ( RT_SUCCESS(rc)
1836	&& !fIsNstGstVmcs)
1837	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1838	}
1839
1840	if (RT_SUCCESS(rc))
1841	{
1842	/*
1843	* Allocate the VM-entry MSR-load area for the guest MSRs.
1844	*
1845	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1846	* the guest and nested-guest.
1847	*/
1848	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1849	&pVmcsInfo->HCPhysGuestMsrLoad);
1850	if (RT_SUCCESS(rc))
1851	{
1852	/*
1853	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1854	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1855	*/
1856	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1857	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1858	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1859
1860	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1861	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1862	&pVmcsInfo->HCPhysHostMsrLoad);
1863	}
1864	}
1865	}
1866	}
1867
1868	return rc;
1869	}
1870
1871
1872	/**
1873	* Free all VT-x structures for the VM.
1874	*
1875	* @returns IPRT status code.
1876	* @param pVM The cross context VM structure.
1877	*/
1878	static void hmR0VmxStructsFree(PVM pVM)
1879	{
1880	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1881	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1882	#endif
1883	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1884
1885	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1886	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1887	{
1888	RTMemFree(pVM->hm.s.vmx.paShadowVmcsFields);
1889	RTMemFree(pVM->hm.s.vmx.paShadowVmcsRoFields);
1890	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjVmreadBitmap, &pVM->hm.s.vmx.pvVmreadBitmap, &pVM->hm.s.vmx.HCPhysVmreadBitmap);
1891	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap, &pVM->hm.s.vmx.HCPhysVmwriteBitmap);
1892	}
1893	#endif
1894
1895	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1896	{
1897	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1898	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1899	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1900	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1901	if (pVM->cpum.ro.GuestFeatures.fVmx)
1902	{
1903	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1904	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1905	}
1906	#endif
1907	}
1908	}
1909
1910
1911	/**
1912	* Allocate all VT-x structures for the VM.
1913	*
1914	* @returns IPRT status code.
1915	* @param pVM The cross context VM structure.
1916	*/
1917	static int hmR0VmxStructsAlloc(PVM pVM)
1918	{
1919	/*
1920	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1921	* The VMCS size cannot be more than 4096 bytes.
1922	*
1923	* See Intel spec. Appendix A.1 "Basic VMX Information".
1924	*/
1925	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1926	if (cbVmcs <= X86_PAGE_4K_SIZE)
1927	{ /* likely */ }
1928	else
1929	{
1930	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1931	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1932	}
1933
1934	/*
1935	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
1936	*/
1937	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1938	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1939	Assert(pVM->hm.s.vmx.pbScratch == NULL);
1940	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1941	#endif
1942
1943	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1944	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1945	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1946
1947	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1948	{
1949	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1950	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1951	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1952	}
1953
1954	/*
1955	* Allocate per-VM VT-x structures.
1956	*/
1957	int rc = VINF_SUCCESS;
1958	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1959	/* Allocate crash-dump magic scratch page. */
1960	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1961	if (RT_FAILURE(rc))
1962	{
1963	hmR0VmxStructsFree(pVM);
1964	return rc;
1965	}
1966	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1967	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1968	#endif
1969
1970	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1971	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1972	{
1973	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1974	&pVM->hm.s.vmx.HCPhysApicAccess);
1975	if (RT_FAILURE(rc))
1976	{
1977	hmR0VmxStructsFree(pVM);
1978	return rc;
1979	}
1980	}
1981
1982	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1983	/* Allocate the shadow VMCS fields array, VMREAD, VMWRITE bitmaps.. */
1984	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1985	{
1986	Assert(!pVM->hm.s.vmx.cShadowVmcsFields);
1987	Assert(!pVM->hm.s.vmx.cShadowVmcsRoFields);
1988	pVM->hm.s.vmx.paShadowVmcsFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
1989	pVM->hm.s.vmx.paShadowVmcsRoFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
1990	if (RT_LIKELY( pVM->hm.s.vmx.paShadowVmcsFields
1991	&& pVM->hm.s.vmx.paShadowVmcsRoFields))
1992	{
1993	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjVmreadBitmap, &pVM->hm.s.vmx.pvVmreadBitmap,
1994	&pVM->hm.s.vmx.HCPhysVmreadBitmap);
1995	if (RT_SUCCESS(rc))
1996	{
1997	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap,
1998	&pVM->hm.s.vmx.HCPhysVmwriteBitmap);
1999	}
2000	}
2001	else
2002	rc = VERR_NO_MEMORY;
2003
2004	if (RT_FAILURE(rc))
2005	{
2006	hmR0VmxStructsFree(pVM);
2007	return rc;
2008	}
2009	}
2010	#endif
2011
2012	/*
2013	* Initialize per-VCPU VT-x structures.
2014	*/
2015	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
2016	{
2017	/* Allocate the guest VMCS structures. */
2018	PVMCPU pVCpu = &pVM->aCpus[idCpu];
2019	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
2020	if (RT_SUCCESS(rc))
2021	{
2022	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2023	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
2024	if (pVM->cpum.ro.GuestFeatures.fVmx)
2025	{
2026	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
2027	if (RT_SUCCESS(rc))
2028	{ /* likely */ }
2029	else
2030	break;
2031	}
2032	#endif
2033	}
2034	else
2035	break;
2036	}
2037
2038	if (RT_FAILURE(rc))
2039	{
2040	hmR0VmxStructsFree(pVM);
2041	return rc;
2042	}
2043
2044	return VINF_SUCCESS;
2045	}
2046
2047	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2048	/**
2049	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
2050	*
2051	* @returns @c true if the MSR is intercepted, @c false otherwise.
2052	* @param pvMsrBitmap The MSR bitmap.
2053	* @param offMsr The MSR byte offset.
2054	* @param iBit The bit offset from the byte offset.
2055	*/
2056	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
2057	{
2058	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
2059	Assert(pbMsrBitmap);
2060	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
2061	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
2062	}
2063	#endif
2064
2065	/**
2066	* Sets the permission bits for the specified MSR in the given MSR bitmap.
2067	*
2068	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
2069	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
2070	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
2071	* the read/write access of this MSR.
2072	*
2073	* @param pVCpu The cross context virtual CPU structure.
2074	* @param pVmcsInfo The VMCS info. object.
2075	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2076	* @param idMsr The MSR value.
2077	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
2078	* include both a read -and- a write permission!
2079	*
2080	* @sa CPUMGetVmxMsrPermission.
2081	* @remarks Can be called with interrupts disabled.
2082	*/
2083	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
2084	{
2085	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
2086	Assert(pbMsrBitmap);
2087	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
2088
2089	/*
2090	* MSR-bitmap Layout:
2091	* Byte index MSR range Interpreted as
2092	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
2093	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
2094	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
2095	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
2096	*
2097	* A bit corresponding to an MSR within the above range causes a VM-exit
2098	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
2099	* the MSR range, it always cause a VM-exit.
2100	*
2101	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
2102	*/
2103	uint16_t const offBitmapRead = 0;
2104	uint16_t const offBitmapWrite = 0x800;
2105	uint16_t offMsr;
2106	int32_t iBit;
2107	if (idMsr <= UINT32_C(0x00001fff))
2108	{
2109	offMsr = 0;
2110	iBit = idMsr;
2111	}
2112	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
2113	{
2114	offMsr = 0x400;
2115	iBit = idMsr - UINT32_C(0xc0000000);
2116	}
2117	else
2118	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
2119
2120	/*
2121	* Set the MSR read permission.
2122	*/
2123	uint16_t const offMsrRead = offBitmapRead + offMsr;
2124	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
2125	if (fMsrpm & VMXMSRPM_ALLOW_RD)
2126	{
2127	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2128	bool const fClear = !fIsNstGstVmcs ? true
2129	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
2130	#else
2131	RT_NOREF2(pVCpu, fIsNstGstVmcs);
2132	bool const fClear = true;
2133	#endif
2134	if (fClear)
2135	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
2136	}
2137	else
2138	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
2139
2140	/*
2141	* Set the MSR write permission.
2142	*/
2143	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
2144	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
2145	if (fMsrpm & VMXMSRPM_ALLOW_WR)
2146	{
2147	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2148	bool const fClear = !fIsNstGstVmcs ? true
2149	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
2150	#else
2151	RT_NOREF2(pVCpu, fIsNstGstVmcs);
2152	bool const fClear = true;
2153	#endif
2154	if (fClear)
2155	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
2156	}
2157	else
2158	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
2159	}
2160
2161
2162	/**
2163	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
2164	* area.
2165	*
2166	* @returns VBox status code.
2167	* @param pVCpu The cross context virtual CPU structure.
2168	* @param pVmcsInfo The VMCS info. object.
2169	* @param cMsrs The number of MSRs.
2170	*/
2171	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
2172	{
2173	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2174	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
2175	if (RT_LIKELY(cMsrs < cMaxSupportedMsrs))
2176	{
2177	/* Commit the MSR counts to the VMCS and update the cache. */
2178	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
2179	{
2180	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
2181	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
2182	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
2183	AssertRCReturn(rc, rc);
2184
2185	pVmcsInfo->cEntryMsrLoad = cMsrs;
2186	pVmcsInfo->cExitMsrStore = cMsrs;
2187	pVmcsInfo->cExitMsrLoad = cMsrs;
2188	}
2189	return VINF_SUCCESS;
2190	}
2191
2192	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u MaxSupported=%u\n", cMsrs, cMaxSupportedMsrs));
2193	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
2194	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2195	}
2196
2197
2198	/**
2199	* Adds a new (or updates the value of an existing) guest/host MSR
2200	* pair to be swapped during the world-switch as part of the
2201	* auto-load/store MSR area in the VMCS.
2202	*
2203	* @returns VBox status code.
2204	* @param pVCpu The cross context virtual CPU structure.
2205	* @param pVmxTransient The VMX-transient structure.
2206	* @param idMsr The MSR.
2207	* @param uGuestMsrValue Value of the guest MSR.
2208	* @param fSetReadWrite Whether to set the guest read/write access of this
2209	* MSR (thus not causing a VM-exit).
2210	* @param fUpdateHostMsr Whether to update the value of the host MSR if
2211	* necessary.
2212	*/
2213	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
2214	bool fSetReadWrite, bool fUpdateHostMsr)
2215	{
2216	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2217	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2218	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2219	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2220	uint32_t i;
2221
2222	/* Paranoia. */
2223	Assert(pGuestMsrLoad);
2224
2225	LogFlowFunc(("pVCpu=%p idMsr=%#RX32 uGestMsrValue=%#RX64\n", pVCpu, idMsr, uGuestMsrValue));
2226
2227	/* Check if the MSR already exists in the VM-entry MSR-load area. */
2228	for (i = 0; i < cMsrs; i++)
2229	{
2230	if (pGuestMsrLoad[i].u32Msr == idMsr)
2231	break;
2232	}
2233
2234	bool fAdded = false;
2235	if (i == cMsrs)
2236	{
2237	/* The MSR does not exist, bump the MSR count to make room for the new MSR. */
2238	++cMsrs;
2239	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2240	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
2241
2242	/* Set the guest to read/write this MSR without causing VM-exits. */
2243	if ( fSetReadWrite
2244	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
2245	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
2246
2247	Log4Func(("Added MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
2248	fAdded = true;
2249	}
2250
2251	/* Update the MSR value for the newly added or already existing MSR. */
2252	pGuestMsrLoad[i].u32Msr = idMsr;
2253	pGuestMsrLoad[i].u64Value = uGuestMsrValue;
2254
2255	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
2256	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2257	{
2258	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2259	pGuestMsrStore[i].u32Msr = idMsr;
2260	pGuestMsrStore[i].u64Value = uGuestMsrValue;
2261	}
2262
2263	/* Update the corresponding slot in the host MSR area. */
2264	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2265	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad);
2266	Assert(pHostMsr != pVmcsInfo->pvGuestMsrStore);
2267	pHostMsr[i].u32Msr = idMsr;
2268
2269	/*
2270	* Only if the caller requests to update the host MSR value AND we've newly added the
2271	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
2272	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
2273	*
2274	* We do this for performance reasons since reading MSRs may be quite expensive.
2275	*/
2276	if (fAdded)
2277	{
2278	if (fUpdateHostMsr)
2279	{
2280	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2281	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2282	pHostMsr[i].u64Value = ASMRdMsr(idMsr);
2283	}
2284	else
2285	{
2286	/* Someone else can do the work. */
2287	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
2288	}
2289	}
2290	return VINF_SUCCESS;
2291	}
2292
2293
2294	/**
2295	* Removes a guest/host MSR pair to be swapped during the world-switch from the
2296	* auto-load/store MSR area in the VMCS.
2297	*
2298	* @returns VBox status code.
2299	* @param pVCpu The cross context virtual CPU structure.
2300	* @param pVmxTransient The VMX-transient structure.
2301	* @param idMsr The MSR.
2302	*/
2303	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
2304	{
2305	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2306	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2307	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2308	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2309
2310	LogFlowFunc(("pVCpu=%p idMsr=%#RX32\n", pVCpu, idMsr));
2311
2312	for (uint32_t i = 0; i < cMsrs; i++)
2313	{
2314	/* Find the MSR. */
2315	if (pGuestMsrLoad[i].u32Msr == idMsr)
2316	{
2317	/*
2318	* If it's the last MSR, we only need to reduce the MSR count.
2319	* If it's -not- the last MSR, copy the last MSR in place of it and reduce the MSR count.
2320	*/
2321	if (i < cMsrs - 1)
2322	{
2323	/* Remove it from the VM-entry MSR-load area. */
2324	pGuestMsrLoad[i].u32Msr = pGuestMsrLoad[cMsrs - 1].u32Msr;
2325	pGuestMsrLoad[i].u64Value = pGuestMsrLoad[cMsrs - 1].u64Value;
2326
2327	/* Remove it from the VM-exit MSR-store area if it's in a different page. */
2328	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2329	{
2330	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2331	Assert(pGuestMsrStore[i].u32Msr == idMsr);
2332	pGuestMsrStore[i].u32Msr = pGuestMsrStore[cMsrs - 1].u32Msr;
2333	pGuestMsrStore[i].u64Value = pGuestMsrStore[cMsrs - 1].u64Value;
2334	}
2335
2336	/* Remove it from the VM-exit MSR-load area. */
2337	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2338	Assert(pHostMsr[i].u32Msr == idMsr);
2339	pHostMsr[i].u32Msr = pHostMsr[cMsrs - 1].u32Msr;
2340	pHostMsr[i].u64Value = pHostMsr[cMsrs - 1].u64Value;
2341	}
2342
2343	/* Reduce the count to reflect the removed MSR and bail. */
2344	--cMsrs;
2345	break;
2346	}
2347	}
2348
2349	/* Update the VMCS if the count changed (meaning the MSR was found and removed). */
2350	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
2351	{
2352	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2353	AssertRCReturn(rc, rc);
2354
2355	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
2356	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2357	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
2358
2359	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
2360	return VINF_SUCCESS;
2361	}
2362
2363	return VERR_NOT_FOUND;
2364	}
2365
2366
2367	/**
2368	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
2369	*
2370	* @returns @c true if found, @c false otherwise.
2371	* @param pVmcsInfo The VMCS info. object.
2372	* @param idMsr The MSR to find.
2373	*/
2374	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
2375	{
2376	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2377	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
2378	Assert(pMsrs);
2379	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
2380	for (uint32_t i = 0; i < cMsrs; i++)
2381	{
2382	if (pMsrs[i].u32Msr == idMsr)
2383	return true;
2384	}
2385	return false;
2386	}
2387
2388
2389	/**
2390	* Updates the value of all host MSRs in the VM-exit MSR-load area.
2391	*
2392	* @param pVCpu The cross context virtual CPU structure.
2393	* @param pVmcsInfo The VMCS info. object.
2394	*
2395	* @remarks No-long-jump zone!!!
2396	*/
2397	static void hmR0VmxUpdateAutoLoadHostMsrs(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2398	{
2399	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2400
2401	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2402	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
2403	Assert(pHostMsrLoad);
2404	Assert(sizeof(pHostMsrLoad) cMsrs <= X86_PAGE_4K_SIZE);
2405	LogFlowFunc(("pVCpu=%p cMsrs=%u\n", pVCpu, cMsrs));
2406	for (uint32_t i = 0; i < cMsrs; i++)
2407	{
2408	/*
2409	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
2410	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
2411	*/
2412	if (pHostMsrLoad[i].u32Msr == MSR_K6_EFER)
2413	pHostMsrLoad[i].u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2414	else
2415	pHostMsrLoad[i].u64Value = ASMRdMsr(pHostMsrLoad[i].u32Msr);
2416	}
2417	}
2418
2419
2420	/**
2421	* Saves a set of host MSRs to allow read/write passthru access to the guest and
2422	* perform lazy restoration of the host MSRs while leaving VT-x.
2423	*
2424	* @param pVCpu The cross context virtual CPU structure.
2425	*
2426	* @remarks No-long-jump zone!!!
2427	*/
2428	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
2429	{
2430	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2431
2432	/*
2433	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
2434	*/
2435	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
2436	{
2437	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
2438	#if HC_ARCH_BITS == 64
2439	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2440	{
2441	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
2442	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
2443	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
2444	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
2445	}
2446	#endif
2447	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
2448	}
2449	}
2450
2451
2452	/**
2453	* Checks whether the MSR belongs to the set of guest MSRs that we restore
2454	* lazily while leaving VT-x.
2455	*
2456	* @returns true if it does, false otherwise.
2457	* @param pVCpu The cross context virtual CPU structure.
2458	* @param idMsr The MSR to check.
2459	*/
2460	static bool hmR0VmxIsLazyGuestMsr(PCVMCPU pVCpu, uint32_t idMsr)
2461	{
2462	NOREF(pVCpu);
2463	#if HC_ARCH_BITS == 64
2464	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2465	{
2466	switch (idMsr)
2467	{
2468	case MSR_K8_LSTAR:
2469	case MSR_K6_STAR:
2470	case MSR_K8_SF_MASK:
2471	case MSR_K8_KERNEL_GS_BASE:
2472	return true;
2473	}
2474	}
2475	#else
2476	RT_NOREF(pVCpu, idMsr);
2477	#endif
2478	return false;
2479	}
2480
2481
2482	/**
2483	* Loads a set of guests MSRs to allow read/passthru to the guest.
2484	*
2485	* The name of this function is slightly confusing. This function does NOT
2486	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
2487	* common prefix for functions dealing with "lazy restoration" of the shared
2488	* MSRs.
2489	*
2490	* @param pVCpu The cross context virtual CPU structure.
2491	*
2492	* @remarks No-long-jump zone!!!
2493	*/
2494	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
2495	{
2496	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2497	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2498
2499	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2500	#if HC_ARCH_BITS == 64
2501	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2502	{
2503	/*
2504	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
2505	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
2506	* we can skip a few MSR writes.
2507	*
2508	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
2509	* guest MSR values in the guest-CPU context might be different to what's currently
2510	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
2511	* CPU, see @bugref{8728}.
2512	*/
2513	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2514	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2515	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
2516	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
2517	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
2518	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
2519	{
2520	#ifdef VBOX_STRICT
2521	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
2522	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
2523	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
2524	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
2525	#endif
2526	}
2527	else
2528	{
2529	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
2530	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2531	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2532	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2533	}
2534	}
2535	#endif
2536	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2537	}
2538
2539
2540	/**
2541	* Performs lazy restoration of the set of host MSRs if they were previously
2542	* loaded with guest MSR values.
2543	*
2544	* @param pVCpu The cross context virtual CPU structure.
2545	*
2546	* @remarks No-long-jump zone!!!
2547	* @remarks The guest MSRs should have been saved back into the guest-CPU
2548	* context by hmR0VmxImportGuestState()!!!
2549	*/
2550	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
2551	{
2552	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2553	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2554
2555	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2556	{
2557	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2558	#if HC_ARCH_BITS == 64
2559	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2560	{
2561	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2562	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2563	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2564	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2565	}
2566	#endif
2567	}
2568	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2569	}
2570
2571
2572	/**
2573	* Verifies that our cached values of the VMCS fields are all consistent with
2574	* what's actually present in the VMCS.
2575	*
2576	* @returns VBox status code.
2577	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2578	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2579	* VMCS content. HMCPU error-field is
2580	* updated, see VMX_VCI_XXX.
2581	* @param pVCpu The cross context virtual CPU structure.
2582	* @param pVmcsInfo The VMCS info. object.
2583	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2584	*/
2585	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2586	{
2587	const char * const pcszVmcs = fIsNstGstVmcs ? "Nested-guest VMCS" : "VMCS";
2588
2589	uint32_t u32Val;
2590	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2591	AssertRCReturn(rc, rc);
2592	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2593	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32EntryCtls, u32Val),
2594	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2595	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2596
2597	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2598	AssertRCReturn(rc, rc);
2599	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2600	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ExitCtls, u32Val),
2601	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2602	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2603
2604	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2605	AssertRCReturn(rc, rc);
2606	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2607	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32PinCtls, u32Val),
2608	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2609	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2610
2611	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2612	AssertRCReturn(rc, rc);
2613	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2614	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls, u32Val),
2615	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2616	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2617
2618	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2619	{
2620	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2621	AssertRCReturn(rc, rc);
2622	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2623	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls2, u32Val),
2624	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2625	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2626	}
2627
2628	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2629	AssertRCReturn(rc, rc);
2630	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2631	("%s exception bitmap mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32XcptBitmap, u32Val),
2632	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2633	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2634
2635	uint64_t u64Val;
2636	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2637	AssertRCReturn(rc, rc);
2638	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2639	("%s TSC offset mismatch: Cache=%#RX64 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64TscOffset, u64Val),
2640	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2641	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2642
2643	NOREF(pcszVmcs);
2644	return VINF_SUCCESS;
2645	}
2646
2647
2648	#ifdef VBOX_STRICT
2649	/**
2650	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2651	*
2652	* @param pVCpu The cross context virtual CPU structure.
2653	* @param pVmcsInfo The VMCS info. object.
2654	*/
2655	static void hmR0VmxCheckHostEferMsr(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2656	{
2657	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2658
2659	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2660	{
2661	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2662	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2663	uint64_t uVmcsEferMsrVmcs;
2664	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2665	AssertRC(rc);
2666
2667	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2668	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2669	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2670	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2671	}
2672	}
2673
2674
2675	/**
2676	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2677	* VMCS are correct.
2678	*
2679	* @param pVCpu The cross context virtual CPU structure.
2680	* @param pVmcsInfo The VMCS info. object.
2681	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2682	*/
2683	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2684	{
2685	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2686
2687	/* Read the various MSR-area counts from the VMCS. */
2688	uint32_t cEntryLoadMsrs;
2689	uint32_t cExitStoreMsrs;
2690	uint32_t cExitLoadMsrs;
2691	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cEntryLoadMsrs); AssertRC(rc);
2692	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cExitStoreMsrs); AssertRC(rc);
2693	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cExitLoadMsrs); AssertRC(rc);
2694
2695	/* Verify all the MSR counts are the same. */
2696	Assert(cEntryLoadMsrs == cExitStoreMsrs);
2697	Assert(cExitStoreMsrs == cExitLoadMsrs);
2698	uint32_t const cMsrs = cExitLoadMsrs;
2699
2700	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2701	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2702
2703	/* Verify the MSR counts are within the allocated page size. */
2704	Assert(sizeof(VMXAUTOMSR) * cMsrs <= X86_PAGE_4K_SIZE);
2705
2706	/* Verify the relevant contents of the MSR areas match. */
2707	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2708	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2709	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2710	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2711	for (uint32_t i = 0; i < cMsrs; i++)
2712	{
2713	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2714	if (fSeparateExitMsrStorePage)
2715	{
2716	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2717	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2718	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2719	}
2720
2721	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2722	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2723	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2724
2725	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2726	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2727	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2728	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2729
2730	/* Verify that cached host EFER MSR matches what's loaded the CPU. */
2731	bool const fIsEferMsr = RT_BOOL(pHostMsrLoad->u32Msr == MSR_K6_EFER);
2732	if (fIsEferMsr)
2733	{
2734	AssertMsgReturnVoid(u64Msr == pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer,
2735	("Cached=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2736	pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer, u64Msr, cMsrs));
2737	}
2738
2739	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2740	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2741	{
2742	uint32_t const fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2743	if (fIsEferMsr)
2744	{
2745	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2746	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2747	}
2748	else
2749	{
2750	if (!fIsNstGstVmcs)
2751	{
2752	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_ALLOW_RD_WR) == VMXMSRPM_ALLOW_RD_WR,
2753	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2754	}
2755	else
2756	{
2757	/*
2758	* A nested-guest VMCS must -also- allow read/write passthrough for the MSR for us to
2759	* execute a nested-guest with MSR passthrough.
2760	*
2761	* Check if the nested-guest MSR bitmap allows passthrough, and if so, assert that we
2762	* allow passthrough too.
2763	*/
2764	void const *pvMsrBitmapNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
2765	Assert(pvMsrBitmapNstGst);
2766	uint32_t const fMsrpmNstGst = CPUMGetVmxMsrPermission(pvMsrBitmapNstGst, pGuestMsrLoad->u32Msr);
2767	AssertMsgReturnVoid(fMsrpm == fMsrpmNstGst,
2768	("u32Msr=%#RX32 cMsrs=%u Permission mismatch fMsrpm=%#x fMsrpmNstGst=%#x!\n",
2769	pGuestMsrLoad->u32Msr, cMsrs, fMsrpm, fMsrpmNstGst));
2770	}
2771	}
2772	}
2773
2774	/* Move to the next MSR. */
2775	pHostMsrLoad++;
2776	pGuestMsrLoad++;
2777	pGuestMsrStore++;
2778	}
2779	}
2780	#endif /* VBOX_STRICT */
2781
2782
2783	/**
2784	* Flushes the TLB using EPT.
2785	*
2786	* @returns VBox status code.
2787	* @param pVCpu The cross context virtual CPU structure of the calling
2788	* EMT. Can be NULL depending on @a enmTlbFlush.
2789	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2790	* enmTlbFlush.
2791	* @param enmTlbFlush Type of flush.
2792	*
2793	* @remarks Caller is responsible for making sure this function is called only
2794	* when NestedPaging is supported and providing @a enmTlbFlush that is
2795	* supported by the CPU.
2796	* @remarks Can be called with interrupts disabled.
2797	*/
2798	static void hmR0VmxFlushEpt(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2799	{
2800	uint64_t au64Descriptor[2];
2801	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2802	au64Descriptor[0] = 0;
2803	else
2804	{
2805	Assert(pVCpu);
2806	Assert(pVmcsInfo);
2807	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2808	}
2809	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2810
2811	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2812	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2813
2814	if ( RT_SUCCESS(rc)
2815	&& pVCpu)
2816	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2817	}
2818
2819
2820	/**
2821	* Flushes the TLB using VPID.
2822	*
2823	* @returns VBox status code.
2824	* @param pVCpu The cross context virtual CPU structure of the calling
2825	* EMT. Can be NULL depending on @a enmTlbFlush.
2826	* @param enmTlbFlush Type of flush.
2827	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2828	* on @a enmTlbFlush).
2829	*
2830	* @remarks Can be called with interrupts disabled.
2831	*/
2832	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2833	{
2834	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2835
2836	uint64_t au64Descriptor[2];
2837	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2838	{
2839	au64Descriptor[0] = 0;
2840	au64Descriptor[1] = 0;
2841	}
2842	else
2843	{
2844	AssertPtr(pVCpu);
2845	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2846	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2847	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2848	au64Descriptor[1] = GCPtr;
2849	}
2850
2851	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2852	AssertMsg(rc == VINF_SUCCESS,
2853	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2854
2855	if ( RT_SUCCESS(rc)
2856	&& pVCpu)
2857	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2858	NOREF(rc);
2859	}
2860
2861
2862	/**
2863	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2864	* otherwise there is nothing really to invalidate.
2865	*
2866	* @returns VBox status code.
2867	* @param pVCpu The cross context virtual CPU structure.
2868	* @param GCVirt Guest virtual address of the page to invalidate.
2869	*/
2870	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
2871	{
2872	AssertPtr(pVCpu);
2873	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2874
2875	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2876	{
2877	/*
2878	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2879	* the EPT case. See @bugref{6043} and @bugref{6177}.
2880	*
2881	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2882	* as this function maybe called in a loop with individual addresses.
2883	*/
2884	PVM pVM = pVCpu->CTX_SUFF(pVM);
2885	if (pVM->hm.s.vmx.fVpid)
2886	{
2887	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2888	if (fVpidFlush)
2889	{
2890	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2891	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2892	}
2893	else
2894	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2895	}
2896	else if (pVM->hm.s.fNestedPaging)
2897	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2898	}
2899
2900	return VINF_SUCCESS;
2901	}
2902
2903
2904	/**
2905	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2906	* case where neither EPT nor VPID is supported by the CPU.
2907	*
2908	* @param pHostCpu The HM physical-CPU structure.
2909	* @param pVCpu The cross context virtual CPU structure.
2910	*
2911	* @remarks Called with interrupts disabled.
2912	*/
2913	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2914	{
2915	AssertPtr(pVCpu);
2916	AssertPtr(pHostCpu);
2917
2918	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2919
2920	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2921	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2922	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2923	pVCpu->hm.s.fForceTLBFlush = false;
2924	return;
2925	}
2926
2927
2928	/**
2929	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2930	*
2931	* @param pHostCpu The HM physical-CPU structure.
2932	* @param pVCpu The cross context virtual CPU structure.
2933	* @param pVmcsInfo The VMCS info. object.
2934	*
2935	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2936	* nomenclature. The reason is, to avoid confusion in compare statements
2937	* since the host-CPU copies are named "ASID".
2938	*
2939	* @remarks Called with interrupts disabled.
2940	*/
2941	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2942	{
2943	#ifdef VBOX_WITH_STATISTICS
2944	bool fTlbFlushed = false;
2945	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2946	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2947	if (!fTlbFlushed) \
2948	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2949	} while (0)
2950	#else
2951	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2952	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2953	#endif
2954
2955	AssertPtr(pVCpu);
2956	AssertPtr(pHostCpu);
2957	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2958
2959	PVM pVM = pVCpu->CTX_SUFF(pVM);
2960	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2961	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2962	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2963
2964	/*
2965	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
2966	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2967	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2968	* cannot reuse the current ASID anymore.
2969	*/
2970	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2971	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2972	{
2973	++pHostCpu->uCurrentAsid;
2974	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2975	{
2976	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2977	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2978	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2979	}
2980
2981	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2982	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2983	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2984
2985	/*
2986	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2987	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2988	*/
2989	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2990	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2991	HMVMX_SET_TAGGED_TLB_FLUSHED();
2992	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2993	}
2994	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2995	{
2996	/*
2997	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2998	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2999	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
3000	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
3001	* mappings, see @bugref{6568}.
3002	*
3003	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
3004	*/
3005	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
3006	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3007	HMVMX_SET_TAGGED_TLB_FLUSHED();
3008	}
3009	else if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
3010	{
3011	/*
3012	* The nested-guest specifies its own guest-physical address to use as the APIC-access
3013	* address which requires flushing the TLB of EPT cached structures.
3014	*
3015	* See Intel spec. 28.3.3.4 "Guidelines for Use of the INVEPT Instruction".
3016	*/
3017	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
3018	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
3019	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
3020	HMVMX_SET_TAGGED_TLB_FLUSHED();
3021	}
3022
3023
3024	pVCpu->hm.s.fForceTLBFlush = false;
3025	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
3026
3027	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
3028	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
3029	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
3030	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
3031	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
3032	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
3033	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
3034	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
3035	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
3036
3037	/* Update VMCS with the VPID. */
3038	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
3039	AssertRC(rc);
3040
3041	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
3042	}
3043
3044
3045	/**
3046	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
3047	*
3048	* @param pHostCpu The HM physical-CPU structure.
3049	* @param pVCpu The cross context virtual CPU structure.
3050	* @param pVmcsInfo The VMCS info. object.
3051	*
3052	* @remarks Called with interrupts disabled.
3053	*/
3054	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
3055	{
3056	AssertPtr(pVCpu);
3057	AssertPtr(pHostCpu);
3058	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3059	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
3060	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
3061
3062	/*
3063	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
3064	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
3065	*/
3066	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3067	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3068	{
3069	pVCpu->hm.s.fForceTLBFlush = true;
3070	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3071	}
3072
3073	/* Check for explicit TLB flushes. */
3074	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3075	{
3076	pVCpu->hm.s.fForceTLBFlush = true;
3077	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3078	}
3079
3080	/* Check for TLB flushes while switching to/from a nested-guest. */
3081	if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
3082	{
3083	pVCpu->hm.s.fForceTLBFlush = true;
3084	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
3085	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
3086	}
3087
3088	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3089	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3090
3091	if (pVCpu->hm.s.fForceTLBFlush)
3092	{
3093	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
3094	pVCpu->hm.s.fForceTLBFlush = false;
3095	}
3096	}
3097
3098
3099	/**
3100	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
3101	*
3102	* @param pHostCpu The HM physical-CPU structure.
3103	* @param pVCpu The cross context virtual CPU structure.
3104	*
3105	* @remarks Called with interrupts disabled.
3106	*/
3107	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
3108	{
3109	AssertPtr(pVCpu);
3110	AssertPtr(pHostCpu);
3111	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3112	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
3113	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
3114
3115	/*
3116	* Force a TLB flush for the first world switch if the current CPU differs from the one we
3117	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
3118	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
3119	* cannot reuse the current ASID anymore.
3120	*/
3121	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3122	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3123	{
3124	pVCpu->hm.s.fForceTLBFlush = true;
3125	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3126	}
3127
3128	/* Check for explicit TLB flushes. */
3129	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3130	{
3131	/*
3132	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
3133	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
3134	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
3135	* include fExplicitFlush's too) - an obscure corner case.
3136	*/
3137	pVCpu->hm.s.fForceTLBFlush = true;
3138	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3139	}
3140
3141	/* Check for TLB flushes while switching to/from a nested-guest. */
3142	if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
3143	{
3144	pVCpu->hm.s.fForceTLBFlush = true;
3145	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
3146	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
3147	}
3148
3149	PVM pVM = pVCpu->CTX_SUFF(pVM);
3150	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3151	if (pVCpu->hm.s.fForceTLBFlush)
3152	{
3153	++pHostCpu->uCurrentAsid;
3154	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
3155	{
3156	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
3157	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
3158	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
3159	}
3160
3161	pVCpu->hm.s.fForceTLBFlush = false;
3162	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3163	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
3164	if (pHostCpu->fFlushAsidBeforeUse)
3165	{
3166	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
3167	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
3168	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
3169	{
3170	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
3171	pHostCpu->fFlushAsidBeforeUse = false;
3172	}
3173	else
3174	{
3175	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
3176	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
3177	}
3178	}
3179	}
3180
3181	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
3182	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
3183	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
3184	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
3185	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
3186	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
3187	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
3188
3189	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
3190	AssertRC(rc);
3191	}
3192
3193
3194	/**
3195	* Flushes the guest TLB entry based on CPU capabilities.
3196	*
3197	* @param pHostCpu The HM physical-CPU structure.
3198	* @param pVCpu The cross context virtual CPU structure.
3199	* @param pVmcsInfo The VMCS info. object.
3200	*
3201	* @remarks Called with interrupts disabled.
3202	*/
3203	static void hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3204	{
3205	#ifdef HMVMX_ALWAYS_FLUSH_TLB
3206	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
3207	#endif
3208	PVM pVM = pVCpu->CTX_SUFF(pVM);
3209	switch (pVM->hm.s.vmx.enmTlbFlushType)
3210	{
3211	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
3212	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
3213	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
3214	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
3215	default:
3216	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
3217	break;
3218	}
3219	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
3220	}
3221
3222
3223	/**
3224	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
3225	* TLB entries from the host TLB before VM-entry.
3226	*
3227	* @returns VBox status code.
3228	* @param pVM The cross context VM structure.
3229	*/
3230	static int hmR0VmxSetupTaggedTlb(PVM pVM)
3231	{
3232	/*
3233	* Determine optimal flush type for nested paging.
3234	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
3235	* unrestricted guest execution (see hmR3InitFinalizeR0()).
3236	*/
3237	if (pVM->hm.s.fNestedPaging)
3238	{
3239	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
3240	{
3241	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
3242	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
3243	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3244	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
3245	else
3246	{
3247	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
3248	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3249	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
3250	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3251	}
3252
3253	/* Make sure the write-back cacheable memory type for EPT is supported. */
3254	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
3255	{
3256	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3257	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
3258	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3259	}
3260
3261	/* EPT requires a page-walk length of 4. */
3262	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
3263	{
3264	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3265	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
3266	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3267	}
3268	}
3269	else
3270	{
3271	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
3272	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3273	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
3274	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3275	}
3276	}
3277
3278	/*
3279	* Determine optimal flush type for VPID.
3280	*/
3281	if (pVM->hm.s.vmx.fVpid)
3282	{
3283	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
3284	{
3285	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
3286	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
3287	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
3288	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
3289	else
3290	{
3291	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
3292	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
3293	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
3294	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
3295	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
3296	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3297	pVM->hm.s.vmx.fVpid = false;
3298	}
3299	}
3300	else
3301	{
3302	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
3303	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
3304	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3305	pVM->hm.s.vmx.fVpid = false;
3306	}
3307	}
3308
3309	/*
3310	* Setup the handler for flushing tagged-TLBs.
3311	*/
3312	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
3313	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
3314	else if (pVM->hm.s.fNestedPaging)
3315	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
3316	else if (pVM->hm.s.vmx.fVpid)
3317	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
3318	else
3319	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
3320	return VINF_SUCCESS;
3321	}
3322
3323
3324	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3325	/**
3326	* Sets up the shadow VMCS fields arrays.
3327	*
3328	* This function builds arrays of VMCS fields to sync the shadow VMCS later while
3329	* executing the guest.
3330	*
3331	* @returns VBox status code.
3332	* @param pVM The cross context VM structure.
3333	*/
3334	static int hmR0VmxSetupShadowVmcsFieldsArrays(PVM pVM)
3335	{
3336	/*
3337	* Paranoia. Ensure we haven't exposed the VMWRITE-All VMX feature to the guest
3338	* when the host does not support it.
3339	*/
3340	bool const fGstVmwriteAll = pVM->cpum.ro.GuestFeatures.fVmxVmwriteAll;
3341	if ( !fGstVmwriteAll
3342	\|\| (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL))
3343	{ /* likely. */ }
3344	else
3345	{
3346	LogRelFunc(("VMX VMWRITE-All feature exposed to the guest but host CPU does not support it!\n"));
3347	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_GST_HOST_VMWRITE_ALL;
3348	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3349	}
3350
3351	uint32_t const cVmcsFields = RT_ELEMENTS(g_aVmcsFields);
3352	uint32_t cRwFields = 0;
3353	uint32_t cRoFields = 0;
3354	for (uint32_t i = 0; i < cVmcsFields; i++)
3355	{
3356	VMXVMCSFIELD VmcsField;
3357	VmcsField.u = g_aVmcsFields[i];
3358
3359	/*
3360	* We will be writing "FULL" (64-bit) fields while syncing the shadow VMCS.
3361	* Therefore, "HIGH" (32-bit portion of 64-bit) fields must not be included
3362	* in the shadow VMCS fields array as they would be redundant.
3363	*
3364	* If the VMCS field depends on a CPU feature that is not exposed to the guest,
3365	* we must not include it in the shadow VMCS fields array. Guests attempting to
3366	* VMREAD/VMWRITE such VMCS fields would cause a VM-exit and we shall emulate
3367	* the required behavior.
3368	*/
3369	if ( VmcsField.n.fAccessType == VMX_VMCSFIELD_ACCESS_FULL
3370	&& CPUMIsGuestVmxVmcsFieldValid(pVM, VmcsField.u))
3371	{
3372	/*
3373	* Read-only fields are placed in a separate array so that while syncing shadow
3374	* VMCS fields later (which is more performance critical) we can avoid branches.
3375	*
3376	* However, if the guest can write to all fields (including read-only fields),
3377	* we treat it a as read/write field. Otherwise, writing to these fields would
3378	* cause a VMWRITE instruction error while syncing the shadow VMCS .
3379	*/
3380	if ( fGstVmwriteAll
3381	\|\| !HMVmxIsVmcsFieldReadOnly(VmcsField.u))
3382	pVM->hm.s.vmx.paShadowVmcsFields[cRwFields++] = VmcsField.u;
3383	else
3384	pVM->hm.s.vmx.paShadowVmcsRoFields[cRoFields++] = VmcsField.u;
3385	}
3386	}
3387
3388	/* Update the counts. */
3389	pVM->hm.s.vmx.cShadowVmcsFields = cRwFields;
3390	pVM->hm.s.vmx.cShadowVmcsRoFields = cRoFields;
3391	return VINF_SUCCESS;
3392	}
3393
3394
3395	/**
3396	* Sets up the VMREAD and VMWRITE bitmaps.
3397	*
3398	* @param pVM The cross context VM structure.
3399	*/
3400	static void hmR0VmxSetupVmreadVmwriteBitmaps(PVM pVM)
3401	{
3402	/*
3403	* By default, ensure guest attempts to acceses to any VMCS fields cause VM-exits.
3404	*/
3405	uint32_t const cbBitmap = X86_PAGE_4K_SIZE;
3406	uint8_t pbVmreadBitmap = (uint8_t )pVM->hm.s.vmx.pvVmreadBitmap;
3407	uint8_t pbVmwriteBitmap = (uint8_t )pVM->hm.s.vmx.pvVmwriteBitmap;
3408	ASMMemFill32(pbVmreadBitmap, cbBitmap, UINT32_C(0xffffffff));
3409	ASMMemFill32(pbVmwriteBitmap, cbBitmap, UINT32_C(0xffffffff));
3410
3411	/*
3412	* Skip intercepting VMREAD/VMWRITE to guest read/write fields in the
3413	* VMREAD and VMWRITE bitmaps.
3414	*/
3415	{
3416	uint32_t const *paShadowVmcsFields = pVM->hm.s.vmx.paShadowVmcsFields;
3417	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
3418	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
3419	{
3420	uint32_t const uVmcsField = paShadowVmcsFields[i];
3421	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
3422	Assert(uVmcsField >> 3 < cbBitmap);
3423	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
3424	ASMBitClear(pbVmwriteBitmap + (uVmcsField >> 3), uVmcsField & 7);
3425	}
3426	}
3427
3428	/*
3429	* Skip intercepting VMREAD for guest read-only fields in the VMREAD bitmap
3430	* if the host supports VMWRITE to all supported VMCS fields.
3431	*/
3432	if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
3433	{
3434	uint32_t const *paShadowVmcsRoFields = pVM->hm.s.vmx.paShadowVmcsRoFields;
3435	uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
3436	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
3437	{
3438	uint32_t const uVmcsField = paShadowVmcsRoFields[i];
3439	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
3440	Assert(uVmcsField >> 3 < cbBitmap);
3441	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
3442	}
3443	}
3444	}
3445	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
3446
3447
3448	/**
3449	* Sets up the virtual-APIC page address for the VMCS.
3450	*
3451	* @returns VBox status code.
3452	* @param pVCpu The cross context virtual CPU structure.
3453	* @param pVmcsInfo The VMCS info. object.
3454	*/
3455	DECLINLINE(int) hmR0VmxSetupVmcsVirtApicAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
3456	{
3457	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3458	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
3459	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
3460	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
3461	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
3462	}
3463
3464
3465	/**
3466	* Sets up the MSR-bitmap address for the VMCS.
3467	*
3468	* @returns VBox status code.
3469	* @param pVCpu The cross context virtual CPU structure.
3470	* @param pVmcsInfo The VMCS info. object.
3471	*/
3472	DECLINLINE(int) hmR0VmxSetupVmcsMsrBitmapAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
3473	{
3474	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3475	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
3476	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
3477	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3478	return VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
3479	}
3480
3481
3482	/**
3483	* Sets up the APIC-access page address for the VMCS.
3484	*
3485	* @returns VBox status code.
3486	* @param pVCpu The cross context virtual CPU structure.
3487	*/
3488	DECLINLINE(int) hmR0VmxSetupVmcsApicAccessAddr(PVMCPU pVCpu)
3489	{
3490	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
3491	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
3492	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
3493	return VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
3494	}
3495
3496
3497	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3498	/**
3499	* Sets up the VMREAD bitmap address for the VMCS.
3500	*
3501	* @returns VBox status code.
3502	* @param pVCpu The cross context virtual CPU structure.
3503	*/
3504	DECLINLINE(int) hmR0VmxSetupVmcsVmreadBitmapAddr(PVMCPU pVCpu)
3505	{
3506	RTHCPHYS const HCPhysVmreadBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmreadBitmap;
3507	Assert(HCPhysVmreadBitmap != NIL_RTHCPHYS);
3508	Assert(!(HCPhysVmreadBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3509	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL, HCPhysVmreadBitmap);
3510	}
3511
3512
3513	/**
3514	* Sets up the VMWRITE bitmap address for the VMCS.
3515	*
3516	* @returns VBox status code.
3517	* @param pVCpu The cross context virtual CPU structure.
3518	*/
3519	DECLINLINE(int) hmR0VmxSetupVmcsVmwriteBitmapAddr(PVMCPU pVCpu)
3520	{
3521	RTHCPHYS const HCPhysVmwriteBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmwriteBitmap;
3522	Assert(HCPhysVmwriteBitmap != NIL_RTHCPHYS);
3523	Assert(!(HCPhysVmwriteBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3524	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL, HCPhysVmwriteBitmap);
3525	}
3526	#endif
3527
3528
3529	/**
3530	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
3531	* in the VMCS.
3532	*
3533	* @returns VBox status code.
3534	* @param pVCpu The cross context virtual CPU structure.
3535	* @param pVmcsInfo The VMCS info. object.
3536	*/
3537	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3538	{
3539	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3540
3541	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
3542	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
3543	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3544
3545	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
3546	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
3547	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
3548
3549	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
3550	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
3551	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3552
3553	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad);
3554	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore);
3555	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad);
3556	AssertRCReturn(rc, rc);
3557	return VINF_SUCCESS;
3558	}
3559
3560
3561	/**
3562	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
3563	*
3564	* @param pVCpu The cross context virtual CPU structure.
3565	* @param pVmcsInfo The VMCS info. object.
3566	*/
3567	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3568	{
3569	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
3570
3571	/*
3572	* The guest can access the following MSRs (read, write) without causing
3573	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
3574	*/
3575	PVM pVM = pVCpu->CTX_SUFF(pVM);
3576	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
3577	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
3578	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
3579	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3580	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
3581
3582	/*
3583	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
3584	* associated with then. We never need to intercept access (writes need to be
3585	* executed without causing a VM-exit, reads will #GP fault anyway).
3586	*
3587	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
3588	* read/write them. We swap the the guest/host MSR value using the
3589	* auto-load/store MSR area.
3590	*/
3591	if (pVM->cpum.ro.GuestFeatures.fIbpb)
3592	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
3593	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
3594	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
3595	if (pVM->cpum.ro.GuestFeatures.fIbrs)
3596	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
3597
3598	#if HC_ARCH_BITS == 64
3599	/*
3600	* Allow full read/write access for the following MSRs (mandatory for VT-x)
3601	* required for 64-bit guests.
3602	*/
3603	if (pVM->hm.s.fAllow64BitGuests)
3604	{
3605	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
3606	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
3607	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
3608	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3609	}
3610	#endif
3611
3612	/*
3613	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
3614	*/
3615	#ifdef VBOX_STRICT
3616	Assert(pVmcsInfo->pvMsrBitmap);
3617	uint32_t const fMsrpmEfer = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
3618	Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
3619	#endif
3620	}
3621
3622
3623	/**
3624	* Sets up pin-based VM-execution controls in the VMCS.
3625	*
3626	* @returns VBox status code.
3627	* @param pVCpu The cross context virtual CPU structure.
3628	* @param pVmcsInfo The VMCS info. object.
3629	*/
3630	static int hmR0VmxSetupVmcsPinCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3631	{
3632	PVM pVM = pVCpu->CTX_SUFF(pVM);
3633	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
3634	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
3635
3636	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
3637	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
3638
3639	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
3640	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
3641
3642	/* Enable the VMX-preemption timer. */
3643	if (pVM->hm.s.vmx.fUsePreemptTimer)
3644	{
3645	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
3646	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
3647	}
3648
3649	#if 0
3650	/* Enable posted-interrupt processing. */
3651	if (pVM->hm.s.fPostedIntrs)
3652	{
3653	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
3654	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
3655	fVal \|= VMX_PIN_CTLS_POSTED_INT;
3656	}
3657	#endif
3658
3659	if ((fVal & fZap) != fVal)
3660	{
3661	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3662	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
3663	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
3664	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3665	}
3666
3667	/* Commit it to the VMCS and update our cache. */
3668	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
3669	AssertRCReturn(rc, rc);
3670	pVmcsInfo->u32PinCtls = fVal;
3671
3672	return VINF_SUCCESS;
3673	}
3674
3675
3676	/**
3677	* Sets up secondary processor-based VM-execution controls in the VMCS.
3678	*
3679	* @returns VBox status code.
3680	* @param pVCpu The cross context virtual CPU structure.
3681	* @param pVmcsInfo The VMCS info. object.
3682	*/
3683	static int hmR0VmxSetupVmcsProcCtls2(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3684	{
3685	PVM pVM = pVCpu->CTX_SUFF(pVM);
3686	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
3687	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3688
3689	/* WBINVD causes a VM-exit. */
3690	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
3691	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
3692
3693	/* Enable EPT (aka nested-paging). */
3694	if (pVM->hm.s.fNestedPaging)
3695	fVal \|= VMX_PROC_CTLS2_EPT;
3696
3697	/* Enable the INVPCID instruction if we expose it to the guest and is supported
3698	by the hardware. Without this, guest executing INVPCID would cause a #UD. */
3699	if ( pVM->cpum.ro.GuestFeatures.fInvpcid
3700	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID))
3701	fVal \|= VMX_PROC_CTLS2_INVPCID;
3702
3703	/* Enable VPID. */
3704	if (pVM->hm.s.vmx.fVpid)
3705	fVal \|= VMX_PROC_CTLS2_VPID;
3706
3707	/* Enable unrestricted guest execution. */
3708	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3709	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
3710
3711	#if 0
3712	if (pVM->hm.s.fVirtApicRegs)
3713	{
3714	/* Enable APIC-register virtualization. */
3715	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
3716	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
3717
3718	/* Enable virtual-interrupt delivery. */
3719	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
3720	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
3721	}
3722	#endif
3723
3724	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is
3725	where the TPR shadow resides. */
3726	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3727	* done dynamically. */
3728	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3729	{
3730	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3731	int rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3732	AssertRCReturn(rc, rc);
3733	}
3734
3735	/* Enable the RDTSCP instruction if we expose it to the guest and is supported
3736	by the hardware. Without this, guest executing RDTSCP would cause a #UD. */
3737	if ( pVM->cpum.ro.GuestFeatures.fRdTscP
3738	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP))
3739	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3740
3741	/* Enable Pause-Loop exiting. */
3742	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
3743	&& pVM->hm.s.vmx.cPleGapTicks
3744	&& pVM->hm.s.vmx.cPleWindowTicks)
3745	{
3746	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3747
3748	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
3749	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
3750	AssertRCReturn(rc, rc);
3751	}
3752
3753	if ((fVal & fZap) != fVal)
3754	{
3755	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3756	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3757	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3758	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3759	}
3760
3761	/* Commit it to the VMCS and update our cache. */
3762	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3763	AssertRCReturn(rc, rc);
3764	pVmcsInfo->u32ProcCtls2 = fVal;
3765
3766	return VINF_SUCCESS;
3767	}
3768
3769
3770	/**
3771	* Sets up processor-based VM-execution controls in the VMCS.
3772	*
3773	* @returns VBox status code.
3774	* @param pVCpu The cross context virtual CPU structure.
3775	* @param pVmcsInfo The VMCS info. object.
3776	*/
3777	static int hmR0VmxSetupVmcsProcCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3778	{
3779	PVM pVM = pVCpu->CTX_SUFF(pVM);
3780
3781	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3782	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3783
3784	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3785	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3786	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3787	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3788	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3789	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3790	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3791
3792	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3793	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3794	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3795	{
3796	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3797	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3798	}
3799
3800	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3801	if (!pVM->hm.s.fNestedPaging)
3802	{
3803	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3804	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3805	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3806	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3807	}
3808
3809	/* Use TPR shadowing if supported by the CPU. */
3810	if ( PDMHasApic(pVM)
3811	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3812	{
3813	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3814	/* CR8 writes cause a VM-exit based on TPR threshold. */
3815	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3816	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3817	int rc = hmR0VmxSetupVmcsVirtApicAddr(pVCpu, pVmcsInfo);
3818	AssertRCReturn(rc, rc);
3819	}
3820	else
3821	{
3822	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3823	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3824	if (pVM->hm.s.fAllow64BitGuests)
3825	{
3826	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3827	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3828	}
3829	}
3830
3831	/* Use MSR-bitmaps if supported by the CPU. */
3832	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3833	{
3834	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3835	int rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3836	AssertRCReturn(rc, rc);
3837	}
3838
3839	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3840	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3841	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3842
3843	if ((fVal & fZap) != fVal)
3844	{
3845	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3846	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3847	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3848	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3849	}
3850
3851	/* Commit it to the VMCS and update our cache. */
3852	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3853	AssertRCReturn(rc, rc);
3854	pVmcsInfo->u32ProcCtls = fVal;
3855
3856	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3857	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3858	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo);
3859
3860	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3861	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3862	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3863
3864	/* Sanity check, should not really happen. */
3865	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3866	{ /* likely */ }
3867	else
3868	{
3869	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3870	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3871	}
3872
3873	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3874	return VINF_SUCCESS;
3875	}
3876
3877
3878	/**
3879	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3880	* Processor-based VM-execution) control fields in the VMCS.
3881	*
3882	* @returns VBox status code.
3883	* @param pVCpu The cross context virtual CPU structure.
3884	* @param pVmcsInfo The VMCS info. object.
3885	*/
3886	static int hmR0VmxSetupVmcsMiscCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3887	{
3888	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3889	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
3890	{
3891	int rc = hmR0VmxSetupVmcsVmreadBitmapAddr(pVCpu);
3892	rc \|= hmR0VmxSetupVmcsVmwriteBitmapAddr(pVCpu);
3893	if (RT_SUCCESS(rc))
3894	{ /* likely */ }
3895	else
3896	{
3897	LogRelFunc(("Failed to setup VMREAD/VMWRITE bitmap addresses. rc=%Rrc\n", rc));
3898	return rc;
3899	}
3900	}
3901	#endif
3902
3903	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
3904	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
3905	if (RT_SUCCESS(rc))
3906	{
3907	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3908	if (RT_SUCCESS(rc))
3909	{
3910	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3911	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3912	rc = VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
3913	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
3914	if (RT_SUCCESS(rc))
3915	{
3916	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3917	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3918	return VINF_SUCCESS;
3919	}
3920	LogRelFunc(("Failed to initialize VMCS CR0/CR4 guest/host mask. rc=%Rrc\n", rc));
3921	}
3922	else
3923	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3924	}
3925	else
3926	LogRelFunc(("Failed to initialize VMCS link pointer. rc=%Rrc\n", rc));
3927	return rc;
3928	}
3929
3930
3931	/**
3932	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3933	*
3934	* We shall setup those exception intercepts that don't change during the
3935	* lifetime of the VM here. The rest are done dynamically while loading the
3936	* guest state.
3937	*
3938	* @returns VBox status code.
3939	* @param pVCpu The cross context virtual CPU structure.
3940	* @param pVmcsInfo The VMCS info. object.
3941	*/
3942	static int hmR0VmxSetupVmcsXcptBitmap(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3943	{
3944	/*
3945	* The following exceptions are always intercepted:
3946	*
3947	* #AC - To prevent the guest from hanging the CPU.
3948	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3949	* recursive #DBs can cause a CPU hang.
3950	* #PF - To sync our shadow page tables when nested-paging is not used.
3951	*/
3952	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3953	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3954	\| RT_BIT(X86_XCPT_DB)
3955	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3956
3957	/* Commit it to the VMCS. */
3958	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3959	AssertRCReturn(rc, rc);
3960
3961	/* Update our cache of the exception bitmap. */
3962	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3963	return VINF_SUCCESS;
3964	}
3965
3966
3967	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3968	/**
3969	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3970	*
3971	* @returns VBox status code.
3972	* @param pVCpu The cross context virtual CPU structure.
3973	* @param pVmcsInfo The VMCS info. object.
3974	*/
3975	static int hmR0VmxSetupVmcsCtlsNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3976	{
3977	PVM pVM = pVCpu->CTX_SUFF(pVM);
3978	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
3979	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
3980	if (RT_SUCCESS(rc))
3981	{
3982	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3983	if (RT_SUCCESS(rc))
3984	{
3985	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3986	rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3987	if (RT_SUCCESS(rc))
3988	return VINF_SUCCESS;
3989	LogRelFunc(("Failed to set up the MSR-bitmap address in the nested-guest VMCS. rc=%Rrc\n", rc));
3990	}
3991	else
3992	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3993	}
3994	else
3995	LogRelFunc(("Failed to set up the auto-load/store MSR addresses in the nested-guest VMCS. rc=%Rrc\n", rc));
3996
3997	return rc;
3998	}
3999	#endif
4000
4001
4002	/**
4003	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
4004	* VMX.
4005	*
4006	* @returns VBox status code.
4007	* @param pVCpu The cross context virtual CPU structure.
4008	* @param pVmcsInfo The VMCS info. object.
4009	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
4010	*/
4011	static int hmR0VmxSetupVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
4012	{
4013	Assert(pVmcsInfo->pvVmcs);
4014	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4015
4016	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
4017	PVM pVM = pVCpu->CTX_SUFF(pVM);
4018	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
4019	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
4020
4021	LogFlowFunc(("\n"));
4022
4023	/*
4024	* Initialize the VMCS using VMCLEAR before loading the VMCS.
4025	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
4026	*/
4027	int rc = hmR0VmxClearVmcs(pVmcsInfo);
4028	if (RT_SUCCESS(rc))
4029	{
4030	rc = hmR0VmxLoadVmcs(pVmcsInfo);
4031	if (RT_SUCCESS(rc))
4032	{
4033	if (!fIsNstGstVmcs)
4034	{
4035	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
4036	if (RT_SUCCESS(rc))
4037	{
4038	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
4039	if (RT_SUCCESS(rc))
4040	{
4041	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
4042	if (RT_SUCCESS(rc))
4043	{
4044	rc = hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
4045	if (RT_SUCCESS(rc))
4046	{
4047	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4048	/*
4049	* If a shadow VMCS is allocated for the VMCS info. object, initialize the
4050	* VMCS revision ID and shadow VMCS indicator bit. Also, clear the VMCS
4051	* making it fit for use when VMCS shadowing is later enabled.
4052	*/
4053	if (pVmcsInfo->pvShadowVmcs)
4054	{
4055	VMXVMCSREVID VmcsRevId;
4056	VmcsRevId.u = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
4057	VmcsRevId.n.fIsShadowVmcs = 1;
4058	(uint32_t )pVmcsInfo->pvShadowVmcs = VmcsRevId.u;
4059	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
4060	if (RT_SUCCESS(rc))
4061	{ /* likely */ }
4062	else
4063	LogRelFunc(("Failed to initialize shadow VMCS. rc=%Rrc\n", rc));
4064	}
4065	#endif
4066	}
4067	else
4068	LogRelFunc(("Failed to initialize exception bitmap. rc=%Rrc\n", rc));
4069	}
4070	else
4071	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
4072	}
4073	else
4074	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
4075	}
4076	else
4077	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
4078	}
4079	else
4080	{
4081	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4082	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
4083	if (RT_SUCCESS(rc))
4084	{ /* likely */ }
4085	else
4086	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
4087	#else
4088	AssertFailed();
4089	#endif
4090	}
4091	}
4092	else
4093	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
4094	}
4095	else
4096	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
4097
4098	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
4099	if (RT_SUCCESS(rc))
4100	{
4101	rc = hmR0VmxClearVmcs(pVmcsInfo);
4102	if (RT_SUCCESS(rc))
4103	{ /* likely */ }
4104	else
4105	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
4106	}
4107
4108	/*
4109	* Update the last-error record both for failures and success, so we
4110	* can propagate the status code back to ring-3 for diagnostics.
4111	*/
4112	hmR0VmxUpdateErrorRecord(pVCpu, rc);
4113	NOREF(pszVmcs);
4114	return rc;
4115	}
4116
4117
4118	/**
4119	* Does global VT-x initialization (called during module initialization).
4120	*
4121	* @returns VBox status code.
4122	*/
4123	VMMR0DECL(int) VMXR0GlobalInit(void)
4124	{
4125	#ifdef HMVMX_USE_FUNCTION_TABLE
4126	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
4127	# ifdef VBOX_STRICT
4128	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
4129	Assert(g_apfnVMExitHandlers[i]);
4130	# endif
4131	#endif
4132	return VINF_SUCCESS;
4133	}
4134
4135
4136	/**
4137	* Does global VT-x termination (called during module termination).
4138	*/
4139	VMMR0DECL(void) VMXR0GlobalTerm()
4140	{
4141	/* Nothing to do currently. */
4142	}
4143
4144
4145	/**
4146	* Sets up and activates VT-x on the current CPU.
4147	*
4148	* @returns VBox status code.
4149	* @param pHostCpu The HM physical-CPU structure.
4150	* @param pVM The cross context VM structure. Can be
4151	* NULL after a host resume operation.
4152	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
4153	* fEnabledByHost is @c true).
4154	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
4155	* @a fEnabledByHost is @c true).
4156	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
4157	* enable VT-x on the host.
4158	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
4159	*/
4160	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
4161	PCSUPHWVIRTMSRS pHwvirtMsrs)
4162	{
4163	AssertPtr(pHostCpu);
4164	AssertPtr(pHwvirtMsrs);
4165	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4166
4167	/* Enable VT-x if it's not already enabled by the host. */
4168	if (!fEnabledByHost)
4169	{
4170	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
4171	if (RT_FAILURE(rc))
4172	return rc;
4173	}
4174
4175	/*
4176	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
4177	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
4178	* invalidated when flushing by VPID.
4179	*/
4180	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
4181	{
4182	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
4183	pHostCpu->fFlushAsidBeforeUse = false;
4184	}
4185	else
4186	pHostCpu->fFlushAsidBeforeUse = true;
4187
4188	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
4189	++pHostCpu->cTlbFlushes;
4190
4191	return VINF_SUCCESS;
4192	}
4193
4194
4195	/**
4196	* Deactivates VT-x on the current CPU.
4197	*
4198	* @returns VBox status code.
4199	* @param pvCpuPage Pointer to the VMXON region.
4200	* @param HCPhysCpuPage Physical address of the VMXON region.
4201	*
4202	* @remarks This function should never be called when SUPR0EnableVTx() or
4203	* similar was used to enable VT-x on the host.
4204	*/
4205	VMMR0DECL(int) VMXR0DisableCpu(void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
4206	{
4207	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
4208
4209	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4210	return hmR0VmxLeaveRootMode();
4211	}
4212
4213
4214	/**
4215	* Does per-VM VT-x initialization.
4216	*
4217	* @returns VBox status code.
4218	* @param pVM The cross context VM structure.
4219	*/
4220	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
4221	{
4222	AssertPtr(pVM);
4223	LogFlowFunc(("pVM=%p\n", pVM));
4224
4225	int rc = hmR0VmxStructsAlloc(pVM);
4226	if (RT_FAILURE(rc))
4227	{
4228	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
4229	return rc;
4230	}
4231
4232	return VINF_SUCCESS;
4233	}
4234
4235
4236	/**
4237	* Does per-VM VT-x termination.
4238	*
4239	* @returns VBox status code.
4240	* @param pVM The cross context VM structure.
4241	*/
4242	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
4243	{
4244	AssertPtr(pVM);
4245	LogFlowFunc(("pVM=%p\n", pVM));
4246
4247	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4248	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
4249	{
4250	Assert(pVM->hm.s.vmx.pvScratch);
4251	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
4252	}
4253	#endif
4254	hmR0VmxStructsFree(pVM);
4255	return VINF_SUCCESS;
4256	}
4257
4258
4259	/**
4260	* Sets up the VM for execution using hardware-assisted VMX.
4261	* This function is only called once per-VM during initialization.
4262	*
4263	* @returns VBox status code.
4264	* @param pVM The cross context VM structure.
4265	*/
4266	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
4267	{
4268	AssertPtr(pVM);
4269	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4270
4271	LogFlowFunc(("pVM=%p\n", pVM));
4272
4273	/*
4274	* At least verify if VMX is enabled, since we can't check if we're in
4275	* VMX root mode or not without causing a #GP.
4276	*/
4277	RTCCUINTREG const uHostCr4 = ASMGetCR4();
4278	if (RT_LIKELY(uHostCr4 & X86_CR4_VMXE))
4279	{ /* likely */ }
4280	else
4281	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
4282
4283	/*
4284	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
4285	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
4286	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
4287	*/
4288	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4289	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
4290	\|\| !pVM->hm.s.vmx.pRealModeTSS))
4291	{
4292	LogRelFunc(("Invalid real-on-v86 state.\n"));
4293	return VERR_INTERNAL_ERROR;
4294	}
4295
4296	/* Initialize these always, see hmR3InitFinalizeR0().*/
4297	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
4298	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
4299
4300	/* Setup the tagged-TLB flush handlers. */
4301	int rc = hmR0VmxSetupTaggedTlb(pVM);
4302	if (RT_FAILURE(rc))
4303	{
4304	LogRelFunc(("Failed to setup tagged TLB. rc=%Rrc\n", rc));
4305	return rc;
4306	}
4307
4308	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4309	/* Setup the shadow VMCS fields array and VMREAD/VMWRITE bitmaps. */
4310	if (pVM->hm.s.vmx.fUseVmcsShadowing)
4311	{
4312	rc = hmR0VmxSetupShadowVmcsFieldsArrays(pVM);
4313	if (RT_SUCCESS(rc))
4314	hmR0VmxSetupVmreadVmwriteBitmaps(pVM);
4315	else
4316	{
4317	LogRelFunc(("Failed to setup shadow VMCS fields arrays. rc=%Rrc\n", rc));
4318	return rc;
4319	}
4320	}
4321	#endif
4322
4323	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
4324	{
4325	PVMCPU pVCpu = &pVM->aCpus[idCpu];
4326	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
4327
4328	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
4329	if (RT_SUCCESS(rc))
4330	{
4331	#if HC_ARCH_BITS == 32
4332	hmR0VmxInitVmcsReadCache(pVCpu);
4333	#endif
4334	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4335	if (pVM->cpum.ro.GuestFeatures.fVmx)
4336	{
4337	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
4338	if (RT_SUCCESS(rc))
4339	{ /* likely */ }
4340	else
4341	{
4342	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
4343	return rc;
4344	}
4345	}
4346	#endif
4347	}
4348	else
4349	{
4350	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
4351	return rc;
4352	}
4353	}
4354
4355	return VINF_SUCCESS;
4356	}
4357
4358
4359	/**
4360	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
4361	* the VMCS.
4362	*
4363	* @returns VBox status code.
4364	*/
4365	static int hmR0VmxExportHostControlRegs(void)
4366	{
4367	RTCCUINTREG uReg = ASMGetCR0();
4368	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
4369	AssertRCReturn(rc, rc);
4370
4371	uReg = ASMGetCR3();
4372	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
4373	AssertRCReturn(rc, rc);
4374
4375	uReg = ASMGetCR4();
4376	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
4377	AssertRCReturn(rc, rc);
4378	return rc;
4379	}
4380
4381
4382	/**
4383	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
4384	* the host-state area in the VMCS.
4385	*
4386	* @returns VBox status code.
4387	* @param pVCpu The cross context virtual CPU structure.
4388	*/
4389	static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
4390	{
4391	#if HC_ARCH_BITS == 64
4392	/**
4393	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
4394	* requirements. See hmR0VmxExportHostSegmentRegs().
4395	*/
4396	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
4397	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
4398	{ \
4399	bool fValidSelector = true; \
4400	if ((selValue) & X86_SEL_LDT) \
4401	{ \
4402	uint32_t uAttr = ASMGetSegAttr((selValue)); \
4403	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
4404	} \
4405	if (fValidSelector) \
4406	{ \
4407	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
4408	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
4409	} \
4410	(selValue) = 0; \
4411	}
4412
4413	/*
4414	* If we've executed guest code using hardware-assisted VMX, the host-state bits
4415	* will be messed up. We should -not- save the messed up state without restoring
4416	* the original host-state, see @bugref{7240}.
4417	*
4418	* This apparently can happen (most likely the FPU changes), deal with it rather than
4419	* asserting. Was observed booting Solaris 10u10 32-bit guest.
4420	*/
4421	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
4422	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
4423	{
4424	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
4425	pVCpu->idCpu));
4426	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
4427	}
4428	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
4429	#else
4430	RT_NOREF(pVCpu);
4431	#endif
4432
4433	/*
4434	* Host DS, ES, FS and GS segment registers.
4435	*/
4436	#if HC_ARCH_BITS == 64
4437	RTSEL uSelDS = ASMGetDS();
4438	RTSEL uSelES = ASMGetES();
4439	RTSEL uSelFS = ASMGetFS();
4440	RTSEL uSelGS = ASMGetGS();
4441	#else
4442	RTSEL uSelDS = 0;
4443	RTSEL uSelES = 0;
4444	RTSEL uSelFS = 0;
4445	RTSEL uSelGS = 0;
4446	#endif
4447
4448	/*
4449	* Host CS and SS segment registers.
4450	*/
4451	RTSEL uSelCS = ASMGetCS();
4452	RTSEL uSelSS = ASMGetSS();
4453
4454	/*
4455	* Host TR segment register.
4456	*/
4457	RTSEL uSelTR = ASMGetTR();
4458
4459	#if HC_ARCH_BITS == 64
4460	/*
4461	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
4462	* gain VM-entry and restore them before we get preempted.
4463	*
4464	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
4465	*/
4466	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
4467	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
4468	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
4469	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
4470	# undef VMXLOCAL_ADJUST_HOST_SEG
4471	#endif
4472
4473	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
4474	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
4475	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
4476	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
4477	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
4478	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
4479	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
4480	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
4481	Assert(uSelCS);
4482	Assert(uSelTR);
4483
4484	/* Write these host selector fields into the host-state area in the VMCS. */
4485	int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
4486	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
4487	#if HC_ARCH_BITS == 64
4488	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
4489	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
4490	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
4491	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
4492	#else
4493	NOREF(uSelDS);
4494	NOREF(uSelES);
4495	NOREF(uSelFS);
4496	NOREF(uSelGS);
4497	#endif
4498	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
4499	AssertRCReturn(rc, rc);
4500
4501	/*
4502	* Host GDTR and IDTR.
4503	*/
4504	RTGDTR Gdtr;
4505	RTIDTR Idtr;
4506	RT_ZERO(Gdtr);
4507	RT_ZERO(Idtr);
4508	ASMGetGDTR(&Gdtr);
4509	ASMGetIDTR(&Idtr);
4510	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
4511	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
4512	AssertRCReturn(rc, rc);
4513
4514	#if HC_ARCH_BITS == 64
4515	/*
4516	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
4517	* them to the maximum limit (0xffff) on every VM-exit.
4518	*/
4519	if (Gdtr.cbGdt != 0xffff)
4520	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
4521
4522	/*
4523	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
4524	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
4525	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
4526	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
4527	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4528	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4529	* at 0xffff on hosts where we are sure it won't cause trouble.
4530	*/
4531	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
4532	if (Idtr.cbIdt < 0x0fff)
4533	# else
4534	if (Idtr.cbIdt != 0xffff)
4535	# endif
4536	{
4537	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4538	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4539	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4540	}
4541	#endif
4542
4543	/*
4544	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4545	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4546	* RPL should be too in most cases.
4547	*/
4548	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4549	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4550
4551	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4552	#if HC_ARCH_BITS == 64
4553	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4554
4555	/*
4556	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4557	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4558	* restoration if the host has something else. Task switching is not supported in 64-bit
4559	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4560	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4561	*
4562	* [1] See Intel spec. 3.5 "System Descriptor Types".
4563	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4564	*/
4565	PVM pVM = pVCpu->CTX_SUFF(pVM);
4566	Assert(pDesc->System.u4Type == 11);
4567	if ( pDesc->System.u16LimitLow != 0x67
4568	\|\| pDesc->System.u4LimitHigh)
4569	{
4570	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4571	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4572	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4573	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4574	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4575	}
4576
4577	/*
4578	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4579	*/
4580	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4581	{
4582	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4583	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4584	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4585	{
4586	/* The GDT is read-only but the writable GDT is available. */
4587	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4588	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4589	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4590	AssertRCReturn(rc, rc);
4591	}
4592	}
4593	#else
4594	uintptr_t const uTRBase = X86DESC_BASE(pDesc);
4595	#endif
4596	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
4597	AssertRCReturn(rc, rc);
4598
4599	/*
4600	* Host FS base and GS base.
4601	*/
4602	#if HC_ARCH_BITS == 64
4603	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4604	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4605	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
4606	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
4607	AssertRCReturn(rc, rc);
4608
4609	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4610	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4611	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4612	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4613	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4614	#endif
4615	return VINF_SUCCESS;
4616	}
4617
4618
4619	/**
4620	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4621	* host-state area of the VMCS.
4622	*
4623	* These MSRs will be automatically restored on the host after every successful
4624	* VM-exit.
4625	*
4626	* @returns VBox status code.
4627	* @param pVCpu The cross context virtual CPU structure.
4628	*
4629	* @remarks No-long-jump zone!!!
4630	*/
4631	static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
4632	{
4633	AssertPtr(pVCpu);
4634
4635	/*
4636	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4637	* rather than swapping them on every VM-entry.
4638	*/
4639	hmR0VmxLazySaveHostMsrs(pVCpu);
4640
4641	/*
4642	* Host Sysenter MSRs.
4643	*/
4644	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
4645	#if HC_ARCH_BITS == 32
4646	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
4647	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
4648	#else
4649	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
4650	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
4651	#endif
4652	AssertRCReturn(rc, rc);
4653
4654	/*
4655	* Host EFER MSR.
4656	*
4657	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4658	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4659	*/
4660	PVM pVM = pVCpu->CTX_SUFF(pVM);
4661	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4662	{
4663	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4664	AssertRCReturn(rc, rc);
4665	}
4666
4667	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4668	* hmR0VmxExportGuestEntryExitCtls(). */
4669
4670	return VINF_SUCCESS;
4671	}
4672
4673
4674	/**
4675	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4676	*
4677	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4678	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4679	*
4680	* @returns true if we need to load guest EFER, false otherwise.
4681	* @param pVCpu The cross context virtual CPU structure.
4682	*
4683	* @remarks Requires EFER, CR4.
4684	* @remarks No-long-jump zone!!!
4685	*/
4686	static bool hmR0VmxShouldSwapEferMsr(PCVMCPU pVCpu)
4687	{
4688	#ifdef HMVMX_ALWAYS_SWAP_EFER
4689	RT_NOREF(pVCpu);
4690	return true;
4691	#else
4692	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4693	PVM pVM = pVCpu->CTX_SUFF(pVM);
4694	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4695	uint64_t const u64GuestEfer = pCtx->msrEFER;
4696
4697	/*
4698	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4699	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4700	*/
4701	if ( CPUMIsGuestInLongModeEx(pCtx)
4702	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4703	return true;
4704
4705	/*
4706	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4707	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4708	*
4709	* See Intel spec. 4.5 "IA-32e Paging".
4710	* See Intel spec. 4.1.1 "Three Paging Modes".
4711	*
4712	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4713	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4714	*/
4715	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4716	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4717	if ( (pCtx->cr4 & X86_CR4_PAE)
4718	&& (pCtx->cr0 & X86_CR0_PG)
4719	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4720	{
4721	/* Assert that host is NX capable. */
4722	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4723	return true;
4724	}
4725
4726	return false;
4727	#endif
4728	}
4729
4730	/**
4731	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4732	* VMCS.
4733	*
4734	* This is typically required when the guest changes paging mode.
4735	*
4736	* @returns VBox status code.
4737	* @param pVCpu The cross context virtual CPU structure.
4738	* @param pVmxTransient The VMX-transient structure.
4739	*
4740	* @remarks Requires EFER.
4741	* @remarks No-long-jump zone!!!
4742	*/
4743	static int hmR0VmxExportGuestEntryExitCtls(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4744	{
4745	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4746	{
4747	PVM pVM = pVCpu->CTX_SUFF(pVM);
4748	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4749
4750	/*
4751	* VM-entry controls.
4752	*/
4753	{
4754	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4755	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4756
4757	/*
4758	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4759	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4760	*
4761	* For nested-guests, this is a mandatory VM-entry control. It's also
4762	* required because we do not want to leak host bits to the nested-guest.
4763	*/
4764	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4765
4766	/*
4767	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4768	*
4769	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4770	* required to get the nested-guest working with hardware-assisted VMX execution.
4771	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a guest hypervisor
4772	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4773	* here rather than while merging the guest VMCS controls.
4774	*/
4775	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4776	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4777	else
4778	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4779
4780	/*
4781	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4782	*
4783	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4784	* regardless of whether the nested-guest VMCS specifies it because we are free to
4785	* load whatever MSRs we require and we do not need to modify the guest visible copy
4786	* of the VM-entry MSR load area.
4787	*/
4788	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4789	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4790	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4791	else
4792	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4793
4794	/*
4795	* The following should -not- be set (since we're not in SMM mode):
4796	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4797	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4798	*/
4799
4800	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4801	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4802
4803	if ((fVal & fZap) == fVal)
4804	{ /* likely */ }
4805	else
4806	{
4807	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
4808	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
4809	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
4810	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4811	}
4812
4813	/* Commit it to the VMCS. */
4814	if (pVmcsInfo->u32EntryCtls != fVal)
4815	{
4816	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
4817	AssertRCReturn(rc, rc);
4818	pVmcsInfo->u32EntryCtls = fVal;
4819	}
4820	}
4821
4822	/*
4823	* VM-exit controls.
4824	*/
4825	{
4826	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4827	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4828
4829	/*
4830	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
4831	* supported the 1-setting of this bit.
4832	*
4833	* For nested-guests, we set the "save debug controls" as the converse
4834	* "load debug controls" is mandatory for nested-guests anyway.
4835	*/
4836	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
4837
4838	/*
4839	* Set the host long mode active (EFER.LMA) bit (which Intel calls
4840	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
4841	* host EFER.LMA and EFER.LME bit to this value. See assertion in
4842	* hmR0VmxExportHostMsrs().
4843	*
4844	* For nested-guests, we always set this bit as we do not support 32-bit
4845	* hosts.
4846	*/
4847	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4848
4849	/*
4850	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
4851	*
4852	* For nested-guests, we should use the "save IA32_EFER" control if we also
4853	* used the "load IA32_EFER" control while exporting VM-entry controls.
4854	*/
4855	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4856	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4857	{
4858	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
4859	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
4860	}
4861
4862	/*
4863	* Enable saving of the VMX-preemption timer value on VM-exit.
4864	* For nested-guests, currently not exposed/used.
4865	*/
4866	if ( pVM->hm.s.vmx.fUsePreemptTimer
4867	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
4868	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
4869
4870	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
4871	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
4872
4873	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
4874	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
4875	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
4876
4877	if ((fVal & fZap) == fVal)
4878	{ /* likely */ }
4879	else
4880	{
4881	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
4882	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
4883	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
4884	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4885	}
4886
4887	/* Commit it to the VMCS. */
4888	if (pVmcsInfo->u32ExitCtls != fVal)
4889	{
4890	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
4891	AssertRCReturn(rc, rc);
4892	pVmcsInfo->u32ExitCtls = fVal;
4893	}
4894	}
4895
4896	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
4897	}
4898	return VINF_SUCCESS;
4899	}
4900
4901
4902	/**
4903	* Sets the TPR threshold in the VMCS.
4904	*
4905	* @returns VBox status code.
4906	* @param pVCpu The cross context virtual CPU structure.
4907	* @param pVmcsInfo The VMCS info. object.
4908	* @param u32TprThreshold The TPR threshold (task-priority class only).
4909	*/
4910	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
4911	{
4912	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
4913	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
4914	RT_NOREF2(pVCpu, pVmcsInfo);
4915	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
4916	}
4917
4918
4919	/**
4920	* Exports the guest APIC TPR state into the VMCS.
4921	*
4922	* @returns VBox status code.
4923	* @param pVCpu The cross context virtual CPU structure.
4924	* @param pVmxTransient The VMX-transient structure.
4925	*
4926	* @remarks No-long-jump zone!!!
4927	*/
4928	static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4929	{
4930	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
4931	{
4932	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
4933
4934	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4935	if (!pVmxTransient->fIsNestedGuest)
4936	{
4937	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
4938	&& APICIsEnabled(pVCpu))
4939	{
4940	/*
4941	* Setup TPR shadowing.
4942	*/
4943	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
4944	{
4945	bool fPendingIntr = false;
4946	uint8_t u8Tpr = 0;
4947	uint8_t u8PendingIntr = 0;
4948	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
4949	AssertRCReturn(rc, rc);
4950
4951	/*
4952	* If there are interrupts pending but masked by the TPR, instruct VT-x to
4953	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
4954	* priority of the pending interrupt so we can deliver the interrupt. If there
4955	* are no interrupts pending, set threshold to 0 to not cause any
4956	* TPR-below-threshold VM-exits.
4957	*/
4958	uint32_t u32TprThreshold = 0;
4959	if (fPendingIntr)
4960	{
4961	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
4962	(which is the Task-Priority Class). */
4963	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
4964	const uint8_t u8TprPriority = u8Tpr >> 4;
4965	if (u8PendingPriority <= u8TprPriority)
4966	u32TprThreshold = u8PendingPriority;
4967	}
4968
4969	rc = hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
4970	AssertRCReturn(rc, rc);
4971	}
4972	}
4973	}
4974	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
4975	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
4976	}
4977	return VINF_SUCCESS;
4978	}
4979
4980
4981	/**
4982	* Gets the guest interruptibility-state.
4983	*
4984	* @returns Guest's interruptibility-state.
4985	* @param pVCpu The cross context virtual CPU structure.
4986	* @param pVmxTransient The VMX-transient structure.
4987	*
4988	* @remarks No-long-jump zone!!!
4989	*/
4990	static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4991	{
4992	/*
4993	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
4994	*/
4995	uint32_t fIntrState = 0;
4996	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4997	{
4998	/* If inhibition is active, RIP and RFLAGS should've been imported from the VMCS already. */
4999	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
5000
5001	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5002	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
5003	{
5004	if (pCtx->eflags.Bits.u1IF)
5005	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
5006	else
5007	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
5008	}
5009	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5010	{
5011	/*
5012	* We can clear the inhibit force flag as even if we go back to the recompiler
5013	* without executing guest code in VT-x, the flag's condition to be cleared is
5014	* met and thus the cleared state is correct.
5015	*/
5016	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5017	}
5018	}
5019
5020	/*
5021	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
5022	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
5023	* setting this would block host-NMIs and IRET will not clear the blocking.
5024	*
5025	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
5026	*
5027	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
5028	*/
5029	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5030	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
5031	&& CPUMIsGuestNmiBlocking(pVCpu))
5032	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
5033
5034	return fIntrState;
5035	}
5036
5037
5038	/**
5039	* Exports the exception intercepts required for guest execution in the VMCS.
5040	*
5041	* @returns VBox status code.
5042	* @param pVCpu The cross context virtual CPU structure.
5043	* @param pVmxTransient The VMX-transient structure.
5044	*
5045	* @remarks No-long-jump zone!!!
5046	*/
5047	static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5048	{
5049	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS)
5050	{
5051	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
5052	if ( !pVmxTransient->fIsNestedGuest
5053	&& pVCpu->hm.s.fGIMTrapXcptUD)
5054	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
5055	else
5056	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
5057
5058	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
5059	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS);
5060	}
5061	return VINF_SUCCESS;
5062	}
5063
5064
5065	/**
5066	* Exports the guest's RIP into the guest-state area in the VMCS.
5067	*
5068	* @returns VBox status code.
5069	* @param pVCpu The cross context virtual CPU structure.
5070	*
5071	* @remarks No-long-jump zone!!!
5072	*/
5073	static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
5074	{
5075	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
5076	{
5077	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
5078
5079	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
5080	AssertRCReturn(rc, rc);
5081
5082	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
5083	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
5084	}
5085	return VINF_SUCCESS;
5086	}
5087
5088
5089	/**
5090	* Exports the guest's RSP into the guest-state area in the VMCS.
5091	*
5092	* @returns VBox status code.
5093	* @param pVCpu The cross context virtual CPU structure.
5094	*
5095	* @remarks No-long-jump zone!!!
5096	*/
5097	static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
5098	{
5099	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
5100	{
5101	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
5102
5103	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
5104	AssertRCReturn(rc, rc);
5105
5106	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
5107	}
5108	return VINF_SUCCESS;
5109	}
5110
5111
5112	/**
5113	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
5114	*
5115	* @returns VBox status code.
5116	* @param pVCpu The cross context virtual CPU structure.
5117	* @param pVmxTransient The VMX-transient structure.
5118	*
5119	* @remarks No-long-jump zone!!!
5120	*/
5121	static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5122	{
5123	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
5124	{
5125	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
5126
5127	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
5128	Let us assert it as such and use 32-bit VMWRITE. */
5129	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
5130	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
5131	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
5132	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
5133
5134	/*
5135	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
5136	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
5137	* can run the real-mode guest code under Virtual 8086 mode.
5138	*/
5139	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5140	if (pVmcsInfo->RealMode.fRealOnV86Active)
5141	{
5142	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5143	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5144	Assert(!pVmxTransient->fIsNestedGuest);
5145	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
5146	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
5147	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
5148	}
5149
5150	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
5151	AssertRCReturn(rc, rc);
5152
5153	/*
5154	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
5155	*
5156	* We must avoid setting any automatic debug exceptions delivery when single-stepping
5157	* through the hypervisor debugger using EFLAGS.TF.
5158	*/
5159	if ( !pVmxTransient->fIsNestedGuest
5160	&& !pVCpu->hm.s.fSingleInstruction
5161	&& fEFlags.Bits.u1TF)
5162	{
5163	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
5164	* premature trips to ring-3 esp since IEM does not yet handle it. */
5165	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
5166	AssertRCReturn(rc, rc);
5167	}
5168	/** @todo NSTVMX: Handling copying of VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS from
5169	* nested-guest VMCS. */
5170
5171	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
5172	Log4Func(("eflags=%#RX32\n", fEFlags.u32));
5173	}
5174	return VINF_SUCCESS;
5175	}
5176
5177
5178	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5179	/**
5180	* Copies the nested-guest VMCS to the shadow VMCS.
5181	*
5182	* @returns VBox status code.
5183	* @param pVCpu The cross context virtual CPU structure.
5184	* @param pVmcsInfo The VMCS info. object.
5185	*
5186	* @remarks No-long-jump zone!!!
5187	*/
5188	static int hmR0VmxCopyNstGstToShadowVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5189	{
5190	PVM pVM = pVCpu->CTX_SUFF(pVM);
5191	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
5192
5193	/*
5194	* Disable interrupts so we don't get preempted while the shadow VMCS is the
5195	* current VMCS, as we may try saving guest lazy MSRs.
5196	*
5197	* Strictly speaking the lazy MSRs are not in the VMCS, but I'd rather not risk
5198	* calling the import VMCS code which is currently performing the guest MSR reads
5199	* (on 64-bit hosts) and accessing the auto-load/store MSR area on 32-bit hosts
5200	* and the rest of the VMX leave session machinery.
5201	*/
5202	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
5203
5204	int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
5205	if (RT_SUCCESS(rc))
5206	{
5207	/*
5208	* Copy all guest read/write VMCS fields.
5209	*
5210	* We don't check for VMWRITE failures here for performance reasons and
5211	* because they are not expected to fail, barring irrecoverable conditions
5212	* like hardware errors.
5213	*/
5214	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
5215	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
5216	{
5217	uint64_t u64Val;
5218	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
5219	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
5220	VMXWriteVmcs64(uVmcsField, u64Val);
5221	}
5222
5223	/*
5224	* If the host CPU supports writing all VMCS fields, copy the guest read-only
5225	* VMCS fields, so the guest can VMREAD them without causing a VM-exit.
5226	*/
5227	if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
5228	{
5229	uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
5230	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
5231	{
5232	uint64_t u64Val;
5233	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsRoFields[i];
5234	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
5235	VMXWriteVmcs64(uVmcsField, u64Val);
5236	}
5237	}
5238
5239	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
5240	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
5241	}
5242
5243	ASMSetFlags(fEFlags);
5244	return rc;
5245	}
5246
5247
5248	/**
5249	* Copies the shadow VMCS to the nested-guest VMCS.
5250	*
5251	* @returns VBox status code.
5252	* @param pVCpu The cross context virtual CPU structure.
5253	* @param pVmcsInfo The VMCS info. object.
5254	*
5255	* @remarks Called with interrupts disabled.
5256	*/
5257	static int hmR0VmxCopyShadowToNstGstVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5258	{
5259	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5260	PVM pVM = pVCpu->CTX_SUFF(pVM);
5261	PVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
5262
5263	int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
5264	if (RT_SUCCESS(rc))
5265	{
5266	/*
5267	* Copy guest read/write fields from the shadow VMCS.
5268	* Guest read-only fields cannot be modified, so no need to copy them.
5269	*
5270	* We don't check for VMREAD failures here for performance reasons and
5271	* because they are not expected to fail, barring irrecoverable conditions
5272	* like hardware errors.
5273	*/
5274	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
5275	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
5276	{
5277	uint64_t u64Val;
5278	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
5279	VMXReadVmcs64(uVmcsField, &u64Val);
5280	IEMWriteVmxVmcsField(pVmcsNstGst, uVmcsField, u64Val);
5281	}
5282
5283	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
5284	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
5285	}
5286	return rc;
5287	}
5288
5289
5290	/**
5291	* Enables VMCS shadowing for the given VMCS info. object.
5292	*
5293	* @param pVCpu The cross context virtual CPU structure.
5294	* @param pVmcsInfo The VMCS info. object.
5295	*
5296	* @remarks No-long-jump zone!!!
5297	*/
5298	static void hmR0VmxEnableVmcsShadowing(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5299	{
5300	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
5301
5302	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
5303	if (!(uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING))
5304	{
5305	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
5306	uProcCtls2 \|= VMX_PROC_CTLS2_VMCS_SHADOWING;
5307	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2);
5308	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, pVmcsInfo->HCPhysShadowVmcs);
5309	AssertRC(rc);
5310
5311	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
5312	pVmcsInfo->u64VmcsLinkPtr = pVmcsInfo->HCPhysShadowVmcs;
5313	Log4Func(("Enabled\n"));
5314	}
5315	}
5316
5317
5318	/**
5319	* Disables VMCS shadowing for the given VMCS info. object.
5320	*
5321	* @param pVCpu The cross context virtual CPU structure.
5322	* @param pVmcsInfo The VMCS info. object.
5323	*
5324	* @remarks No-long-jump zone!!!
5325	*/
5326	static void hmR0VmxDisableVmcsShadowing(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5327	{
5328	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
5329
5330	/*
5331	* We want all VMREAD and VMWRITE instructions to cause VM-exits, so we clear the
5332	* VMCS shadowing control. However, VM-entry requires the shadow VMCS indicator bit
5333	* to match the VMCS shadowing control if the VMCS link pointer is not NIL_RTHCPHYS.
5334	* Hence, we must also reset the VMCS link pointer to ensure VM-entry does not fail.
5335	*
5336	* See Intel spec. 26.2.1.1 "VM-Execution Control Fields".
5337	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5338	*/
5339	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
5340	if (uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
5341	{
5342	uProcCtls2 &= ~VMX_PROC_CTLS2_VMCS_SHADOWING;
5343	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2);
5344	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
5345	AssertRC(rc);
5346
5347	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
5348	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
5349	Log4Func(("Disabled\n"));
5350	}
5351	}
5352	#endif
5353
5354
5355	/**
5356	* Exports the guest hardware-virtualization state.
5357	*
5358	* @returns VBox status code.
5359	* @param pVCpu The cross context virtual CPU structure.
5360	* @param pVmxTransient The VMX-transient structure.
5361	*
5362	* @remarks No-long-jump zone!!!
5363	*/
5364	static int hmR0VmxExportGuestHwvirtState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5365	{
5366	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_HWVIRT)
5367	{
5368	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5369	/*
5370	* Check if the VMX feature is exposed to the guest and if the host CPU supports
5371	* VMCS shadowing.
5372	*/
5373	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
5374	{
5375	/*
5376	* If the guest hypervisor has loaded a current VMCS and is in VMX root mode,
5377	* copy the guest hypervisor's current VMCS into the shadow VMCS and enable
5378	* VMCS shadowing to skip intercepting some or all VMREAD/VMWRITE VM-exits.
5379	*
5380	* We check for VMX root mode here in case the guest executes VMXOFF without
5381	* clearing the current VMCS pointer and our VMXOFF instruction emulation does
5382	* not clear the current VMCS pointer.
5383	*/
5384	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5385	if ( CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx)
5386	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)
5387	&& CPUMIsGuestVmxCurrentVmcsValid(pVCpu, &pVCpu->cpum.GstCtx))
5388	{
5389	/* Paranoia. */
5390	Assert(!pVmxTransient->fIsNestedGuest);
5391
5392	/*
5393	* For performance reasons, also check if the guest hypervisor's current VMCS
5394	* was newly loaded or modified before copying it to the shadow VMCS.
5395	*/
5396	if (!pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs)
5397	{
5398	int rc = hmR0VmxCopyNstGstToShadowVmcs(pVCpu, pVmcsInfo);
5399	AssertRCReturn(rc, rc);
5400	pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs = true;
5401	}
5402	hmR0VmxEnableVmcsShadowing(pVCpu, pVmcsInfo);
5403	}
5404	else
5405	hmR0VmxDisableVmcsShadowing(pVCpu, pVmcsInfo);
5406	}
5407	#else
5408	NOREF(pVmxTransient);
5409	#endif
5410	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_HWVIRT);
5411	}
5412	return VINF_SUCCESS;
5413	}
5414
5415
5416	/**
5417	* Exports the guest CR0 control register into the guest-state area in the VMCS.
5418	*
5419	* The guest FPU state is always pre-loaded hence we don't need to bother about
5420	* sharing FPU related CR0 bits between the guest and host.
5421	*
5422	* @returns VBox status code.
5423	* @param pVCpu The cross context virtual CPU structure.
5424	* @param pVmxTransient The VMX-transient structure.
5425	*
5426	* @remarks No-long-jump zone!!!
5427	*/
5428	static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5429	{
5430	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
5431	{
5432	PVM pVM = pVCpu->CTX_SUFF(pVM);
5433	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5434
5435	/*
5436	* Figure out fixed CR0 bits in VMX operation.
5437	*/
5438	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
5439	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
5440	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
5441	if (pVM->hm.s.vmx.fUnrestrictedGuest)
5442	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
5443	else
5444	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
5445
5446	if (!pVmxTransient->fIsNestedGuest)
5447	{
5448	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5449	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5450	uint64_t const u64ShadowCr0 = u64GuestCr0;
5451	Assert(!RT_HI_U32(u64GuestCr0));
5452
5453	/*
5454	* Setup VT-x's view of the guest CR0.
5455	*/
5456	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5457	if (pVM->hm.s.fNestedPaging)
5458	{
5459	if (CPUMIsGuestPagingEnabled(pVCpu))
5460	{
5461	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
5462	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
5463	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
5464	}
5465	else
5466	{
5467	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
5468	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
5469	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
5470	}
5471
5472	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
5473	if (pVM->hm.s.vmx.fUnrestrictedGuest)
5474	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
5475	}
5476	else
5477	{
5478	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
5479	u64GuestCr0 \|= X86_CR0_WP;
5480	}
5481
5482	/*
5483	* Guest FPU bits.
5484	*
5485	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
5486	* using CR0.TS.
5487	*
5488	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
5489	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
5490	*/
5491	u64GuestCr0 \|= X86_CR0_NE;
5492
5493	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
5494	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
5495
5496	/*
5497	* Update exception intercepts.
5498	*/
5499	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
5500	if (pVmcsInfo->RealMode.fRealOnV86Active)
5501	{
5502	Assert(PDMVmmDevHeapIsEnabled(pVM));
5503	Assert(pVM->hm.s.vmx.pRealModeTSS);
5504	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
5505	}
5506	else
5507	{
5508	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
5509	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
5510	if (fInterceptMF)
5511	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
5512	}
5513
5514	/* Additional intercepts for debugging, define these yourself explicitly. */
5515	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
5516	uXcptBitmap \|= 0
5517	\| RT_BIT(X86_XCPT_BP)
5518	\| RT_BIT(X86_XCPT_DE)
5519	\| RT_BIT(X86_XCPT_NM)
5520	\| RT_BIT(X86_XCPT_TS)
5521	\| RT_BIT(X86_XCPT_UD)
5522	\| RT_BIT(X86_XCPT_NP)
5523	\| RT_BIT(X86_XCPT_SS)
5524	\| RT_BIT(X86_XCPT_GP)
5525	\| RT_BIT(X86_XCPT_PF)
5526	\| RT_BIT(X86_XCPT_MF)
5527	;
5528	#elif defined(HMVMX_ALWAYS_TRAP_PF)
5529	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
5530	#endif
5531	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
5532	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
5533	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
5534
5535	/* Apply the hardware specified fixed CR0 bits and enable caching. */
5536	u64GuestCr0 \|= fSetCr0;
5537	u64GuestCr0 &= fZapCr0;
5538	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5539
5540	/* Commit the CR0 and related fields to the guest VMCS. */
5541	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0); /** @todo Fix to 64-bit when we drop 32-bit. */
5542	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
5543	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5544	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5545	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
5546	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
5547	AssertRCReturn(rc, rc);
5548
5549	/* Update our caches. */
5550	pVmcsInfo->u32ProcCtls = uProcCtls;
5551	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
5552
5553	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5554	}
5555	else
5556	{
5557	/*
5558	* With nested-guests, we may have extended the guest/host mask here since we
5559	* merged in the outer guest's mask. Thus, the merged mask can include more bits
5560	* (to read from the nested-guest CR0 read-shadow) than the guest hypervisor
5561	* originally supplied. We must copy those bits from the nested-guest CR0 into
5562	* the nested-guest CR0 read-shadow.
5563	*/
5564	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5565	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5566	uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(pVCpu, &pVCpu->cpum.GstCtx, pVmcsInfo->u64Cr0Mask);
5567	Assert(!RT_HI_U32(u64GuestCr0));
5568	Assert(u64GuestCr0 & X86_CR0_NE);
5569
5570	/*
5571	* Apply the hardware specified fixed CR0 bits and enable caching.
5572	* Note! We could be altering our VMX emulation's fixed bits. We thus
5573	* need to re-apply them while importing CR0.
5574	*/
5575	u64GuestCr0 \|= fSetCr0;
5576	u64GuestCr0 &= fZapCr0;
5577	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5578
5579	/* Commit the CR0 and CR0 read-shadow to the nested-guest VMCS. */
5580	/** @todo NSTVMX: Fix to 64-bit when we drop 32-bit. */
5581	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0);
5582	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
5583	AssertRCReturn(rc, rc);
5584
5585	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5586	}
5587
5588	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
5589	}
5590
5591	return VINF_SUCCESS;
5592	}
5593
5594
5595	/**
5596	* Exports the guest control registers (CR3, CR4) into the guest-state area
5597	* in the VMCS.
5598	*
5599	* @returns VBox strict status code.
5600	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
5601	* without unrestricted guest access and the VMMDev is not presently
5602	* mapped (e.g. EFI32).
5603	*
5604	* @param pVCpu The cross context virtual CPU structure.
5605	* @param pVmxTransient The VMX-transient structure.
5606	*
5607	* @remarks No-long-jump zone!!!
5608	*/
5609	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5610	{
5611	int rc = VINF_SUCCESS;
5612	PVM pVM = pVCpu->CTX_SUFF(pVM);
5613
5614	/*
5615	* Guest CR2.
5616	* It's always loaded in the assembler code. Nothing to do here.
5617	*/
5618
5619	/*
5620	* Guest CR3.
5621	*/
5622	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
5623	{
5624	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
5625
5626	RTGCPHYS GCPhysGuestCr3 = NIL_RTGCPHYS;
5627	if (pVM->hm.s.fNestedPaging)
5628	{
5629	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5630	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
5631
5632	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
5633	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
5634	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
5635	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
5636
5637	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
5638	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
5639	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
5640
5641	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
5642	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
5643	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
5644	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
5645	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
5646	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
5647	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
5648
5649	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
5650	AssertRCReturn(rc, rc);
5651
5652	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5653	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5654	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
5655	{
5656	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
5657	if (CPUMIsGuestInPAEModeEx(pCtx))
5658	{
5659	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5660	AssertRCReturn(rc, rc);
5661	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
5662	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
5663	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
5664	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
5665	AssertRCReturn(rc, rc);
5666	}
5667
5668	/*
5669	* The guest's view of its CR3 is unblemished with nested paging when the
5670	* guest is using paging or we have unrestricted guest execution to handle
5671	* the guest when it's not using paging.
5672	*/
5673	GCPhysGuestCr3 = pCtx->cr3;
5674	}
5675	else
5676	{
5677	/*
5678	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
5679	* thinks it accesses physical memory directly, we use our identity-mapped
5680	* page table to map guest-linear to guest-physical addresses. EPT takes care
5681	* of translating it to host-physical addresses.
5682	*/
5683	RTGCPHYS GCPhys;
5684	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
5685
5686	/* We obtain it here every time as the guest could have relocated this PCI region. */
5687	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
5688	if (RT_SUCCESS(rc))
5689	{ /* likely */ }
5690	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
5691	{
5692	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
5693	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
5694	}
5695	else
5696	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
5697
5698	GCPhysGuestCr3 = GCPhys;
5699	}
5700
5701	Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCr3));
5702	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCr3);
5703	AssertRCReturn(rc, rc);
5704	}
5705	else
5706	{
5707	/* Non-nested paging case, just use the hypervisor's CR3. */
5708	RTHCPHYS const HCPhysGuestCr3 = PGMGetHyperCR3(pVCpu);
5709
5710	Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCr3));
5711	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCr3);
5712	AssertRCReturn(rc, rc);
5713	}
5714
5715	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
5716	}
5717
5718	/*
5719	* Guest CR4.
5720	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
5721	*/
5722	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
5723	{
5724	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5725	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5726
5727	/*
5728	* Figure out fixed CR4 bits in VMX operation.
5729	*/
5730	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
5731	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5732	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5733
5734	/*
5735	* With nested-guests, we may have extended the guest/host mask here (since we
5736	* merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
5737	* mask can include more bits (to read from the nested-guest CR4 read-shadow) than
5738	* the guest hypervisor originally supplied. Thus, we should, in essence, copy
5739	* those bits from the nested-guest CR4 into the nested-guest CR4 read-shadow.
5740	*/
5741	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
5742	uint64_t u64GuestCr4 = pCtx->cr4;
5743	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest
5744	? pCtx->cr4
5745	: CPUMGetGuestVmxMaskedCr4(pVCpu, pCtx, pVmcsInfo->u64Cr4Mask);
5746	Assert(!RT_HI_U32(u64GuestCr4));
5747
5748	/*
5749	* Setup VT-x's view of the guest CR4.
5750	*
5751	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
5752	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
5753	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
5754	*
5755	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
5756	*/
5757	if (pVmcsInfo->RealMode.fRealOnV86Active)
5758	{
5759	Assert(pVM->hm.s.vmx.pRealModeTSS);
5760	Assert(PDMVmmDevHeapIsEnabled(pVM));
5761	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5762	}
5763
5764	if (pVM->hm.s.fNestedPaging)
5765	{
5766	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5767	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
5768	{
5769	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5770	u64GuestCr4 \|= X86_CR4_PSE;
5771	/* Our identity mapping is a 32-bit page directory. */
5772	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5773	}
5774	/* else use guest CR4.*/
5775	}
5776	else
5777	{
5778	Assert(!pVmxTransient->fIsNestedGuest);
5779
5780	/*
5781	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
5782	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
5783	*/
5784	switch (pVCpu->hm.s.enmShadowMode)
5785	{
5786	case PGMMODE_REAL: /* Real-mode. */
5787	case PGMMODE_PROTECTED: /* Protected mode without paging. */
5788	case PGMMODE_32_BIT: /* 32-bit paging. */
5789	{
5790	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5791	break;
5792	}
5793
5794	case PGMMODE_PAE: /* PAE paging. */
5795	case PGMMODE_PAE_NX: /* PAE paging with NX. */
5796	{
5797	u64GuestCr4 \|= X86_CR4_PAE;
5798	break;
5799	}
5800
5801	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
5802	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
5803	#ifdef VBOX_WITH_64_BITS_GUESTS
5804	break;
5805	#endif
5806	default:
5807	AssertFailed();
5808	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5809	}
5810	}
5811
5812	/*
5813	* Apply the hardware specified fixed CR4 bits (mainly CR4.VMXE).
5814	* Note! For nested-guests, we could be altering our VMX emulation's
5815	* fixed bits. We thus need to re-apply them while importing CR4.
5816	*/
5817	u64GuestCr4 \|= fSetCr4;
5818	u64GuestCr4 &= fZapCr4;
5819
5820	/* Commit the CR4 and CR4 read-shadow to the guest VMCS. */
5821	/** @todo Fix to 64-bit when we drop 32-bit. */
5822	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u64GuestCr4);
5823	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4);
5824	AssertRCReturn(rc, rc);
5825
5826	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
5827	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
5828
5829	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
5830
5831	Log4Func(("cr4=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
5832	}
5833	return rc;
5834	}
5835
5836
5837	/**
5838	* Exports the guest debug registers into the guest-state area in the VMCS.
5839	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
5840	*
5841	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
5842	*
5843	* @returns VBox status code.
5844	* @param pVCpu The cross context virtual CPU structure.
5845	* @param pVmxTransient The VMX-transient structure.
5846	*
5847	* @remarks No-long-jump zone!!!
5848	*/
5849	static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5850	{
5851	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5852
5853	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
5854	* stepping. */
5855	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5856	if (pVmxTransient->fIsNestedGuest)
5857	{
5858	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
5859	AssertRCReturn(rc, rc);
5860
5861	/* Always intercept Mov DRx accesses for the nested-guest for now. */
5862	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5863	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
5864	AssertRCReturn(rc, rc);
5865	return VINF_SUCCESS;
5866	}
5867
5868	#ifdef VBOX_STRICT
5869	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
5870	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
5871	{
5872	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
5873	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
5874	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
5875	}
5876	#endif
5877
5878	bool fSteppingDB = false;
5879	bool fInterceptMovDRx = false;
5880	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5881	if (pVCpu->hm.s.fSingleInstruction)
5882	{
5883	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
5884	PVM pVM = pVCpu->CTX_SUFF(pVM);
5885	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
5886	{
5887	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
5888	Assert(fSteppingDB == false);
5889	}
5890	else
5891	{
5892	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
5893	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
5894	pVCpu->hm.s.fClearTrapFlag = true;
5895	fSteppingDB = true;
5896	}
5897	}
5898
5899	uint32_t u32GuestDr7;
5900	if ( fSteppingDB
5901	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
5902	{
5903	/*
5904	* Use the combined guest and host DRx values found in the hypervisor register set
5905	* because the hypervisor debugger has breakpoints active or someone is single stepping
5906	* on the host side without a monitor trap flag.
5907	*
5908	* Note! DBGF expects a clean DR6 state before executing guest code.
5909	*/
5910	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5911	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5912	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
5913	{
5914	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5915	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
5916	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
5917	}
5918	else
5919	#endif
5920	if (!CPUMIsHyperDebugStateActive(pVCpu))
5921	{
5922	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5923	Assert(CPUMIsHyperDebugStateActive(pVCpu));
5924	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5925	}
5926
5927	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
5928	u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
5929	pVCpu->hm.s.fUsingHyperDR7 = true;
5930	fInterceptMovDRx = true;
5931	}
5932	else
5933	{
5934	/*
5935	* If the guest has enabled debug registers, we need to load them prior to
5936	* executing guest code so they'll trigger at the right time.
5937	*/
5938	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
5939	{
5940	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5941	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5942	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
5943	{
5944	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5945	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
5946	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
5947	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5948	}
5949	else
5950	#endif
5951	if (!CPUMIsGuestDebugStateActive(pVCpu))
5952	{
5953	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5954	Assert(CPUMIsGuestDebugStateActive(pVCpu));
5955	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5956	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5957	}
5958	Assert(!fInterceptMovDRx);
5959	}
5960	/*
5961	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
5962	* must intercept #DB in order to maintain a correct DR6 guest value, and
5963	* because we need to intercept it to prevent nested #DBs from hanging the
5964	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
5965	*/
5966	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5967	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
5968	&& !CPUMIsGuestDebugStateActive(pVCpu))
5969	#else
5970	else if (!CPUMIsGuestDebugStateActive(pVCpu))
5971	#endif
5972	{
5973	fInterceptMovDRx = true;
5974	}
5975
5976	/* Update DR7 with the actual guest value. */
5977	u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
5978	pVCpu->hm.s.fUsingHyperDR7 = false;
5979	}
5980
5981	if (fInterceptMovDRx)
5982	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5983	else
5984	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
5985
5986	/*
5987	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
5988	* monitor-trap flag and update our cache.
5989	*/
5990	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5991	{
5992	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5993	AssertRCReturn(rc2, rc2);
5994	pVmcsInfo->u32ProcCtls = uProcCtls;
5995	}
5996
5997	/*
5998	* Update guest DR7.
5999	*/
6000	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
6001	AssertRCReturn(rc, rc);
6002
6003	/*
6004	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
6005	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
6006	*
6007	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
6008	*/
6009	if (fSteppingDB)
6010	{
6011	Assert(pVCpu->hm.s.fSingleInstruction);
6012	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
6013
6014	uint32_t fIntrState = 0;
6015	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
6016	AssertRCReturn(rc, rc);
6017
6018	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
6019	{
6020	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
6021	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
6022	AssertRCReturn(rc, rc);
6023	}
6024	}
6025
6026	return VINF_SUCCESS;
6027	}
6028
6029
6030	#ifdef VBOX_STRICT
6031	/**
6032	* Strict function to validate segment registers.
6033	*
6034	* @param pVCpu The cross context virtual CPU structure.
6035	* @param pVmcsInfo The VMCS info. object.
6036	*
6037	* @remarks Will import guest CR0 on strict builds during validation of
6038	* segments.
6039	*/
6040	static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
6041	{
6042	/*
6043	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6044	*
6045	* The reason we check for attribute value 0 in this function and not just the unusable bit is
6046	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
6047	* unusable bit and doesn't change the guest-context value.
6048	*/
6049	PVM pVM = pVCpu->CTX_SUFF(pVM);
6050	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6051	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
6052	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6053	&& ( !CPUMIsGuestInRealModeEx(pCtx)
6054	&& !CPUMIsGuestInV86ModeEx(pCtx)))
6055	{
6056	/* Protected mode checks */
6057	/* CS */
6058	Assert(pCtx->cs.Attr.n.u1Present);
6059	Assert(!(pCtx->cs.Attr.u & 0xf00));
6060	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
6061	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
6062	\|\| !(pCtx->cs.Attr.n.u1Granularity));
6063	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
6064	\|\| (pCtx->cs.Attr.n.u1Granularity));
6065	/* CS cannot be loaded with NULL in protected mode. */
6066	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
6067	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
6068	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
6069	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
6070	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
6071	else
6072	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
6073	/* SS */
6074	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
6075	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
6076	if ( !(pCtx->cr0 & X86_CR0_PE)
6077	\|\| pCtx->cs.Attr.n.u4Type == 3)
6078	{
6079	Assert(!pCtx->ss.Attr.n.u2Dpl);
6080	}
6081	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
6082	{
6083	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
6084	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
6085	Assert(pCtx->ss.Attr.n.u1Present);
6086	Assert(!(pCtx->ss.Attr.u & 0xf00));
6087	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
6088	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
6089	\|\| !(pCtx->ss.Attr.n.u1Granularity));
6090	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
6091	\|\| (pCtx->ss.Attr.n.u1Granularity));
6092	}
6093	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
6094	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
6095	{
6096	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6097	Assert(pCtx->ds.Attr.n.u1Present);
6098	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
6099	Assert(!(pCtx->ds.Attr.u & 0xf00));
6100	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
6101	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
6102	\|\| !(pCtx->ds.Attr.n.u1Granularity));
6103	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
6104	\|\| (pCtx->ds.Attr.n.u1Granularity));
6105	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6106	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
6107	}
6108	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
6109	{
6110	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6111	Assert(pCtx->es.Attr.n.u1Present);
6112	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
6113	Assert(!(pCtx->es.Attr.u & 0xf00));
6114	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
6115	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
6116	\|\| !(pCtx->es.Attr.n.u1Granularity));
6117	Assert( !(pCtx->es.u32Limit & 0xfff00000)
6118	\|\| (pCtx->es.Attr.n.u1Granularity));
6119	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6120	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
6121	}
6122	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
6123	{
6124	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6125	Assert(pCtx->fs.Attr.n.u1Present);
6126	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
6127	Assert(!(pCtx->fs.Attr.u & 0xf00));
6128	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
6129	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
6130	\|\| !(pCtx->fs.Attr.n.u1Granularity));
6131	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
6132	\|\| (pCtx->fs.Attr.n.u1Granularity));
6133	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6134	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
6135	}
6136	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
6137	{
6138	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6139	Assert(pCtx->gs.Attr.n.u1Present);
6140	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
6141	Assert(!(pCtx->gs.Attr.u & 0xf00));
6142	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
6143	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
6144	\|\| !(pCtx->gs.Attr.n.u1Granularity));
6145	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
6146	\|\| (pCtx->gs.Attr.n.u1Granularity));
6147	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6148	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
6149	}
6150	/* 64-bit capable CPUs. */
6151	# if HC_ARCH_BITS == 64
6152	Assert(!RT_HI_U32(pCtx->cs.u64Base));
6153	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
6154	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
6155	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
6156	# endif
6157	}
6158	else if ( CPUMIsGuestInV86ModeEx(pCtx)
6159	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
6160	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
6161	{
6162	/* Real and v86 mode checks. */
6163	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
6164	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
6165	if (pVmcsInfo->RealMode.fRealOnV86Active)
6166	{
6167	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
6168	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
6169	}
6170	else
6171	{
6172	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
6173	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
6174	}
6175
6176	/* CS */
6177	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
6178	Assert(pCtx->cs.u32Limit == 0xffff);
6179	Assert(u32CSAttr == 0xf3);
6180	/* SS */
6181	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
6182	Assert(pCtx->ss.u32Limit == 0xffff);
6183	Assert(u32SSAttr == 0xf3);
6184	/* DS */
6185	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
6186	Assert(pCtx->ds.u32Limit == 0xffff);
6187	Assert(u32DSAttr == 0xf3);
6188	/* ES */
6189	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
6190	Assert(pCtx->es.u32Limit == 0xffff);
6191	Assert(u32ESAttr == 0xf3);
6192	/* FS */
6193	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
6194	Assert(pCtx->fs.u32Limit == 0xffff);
6195	Assert(u32FSAttr == 0xf3);
6196	/* GS */
6197	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
6198	Assert(pCtx->gs.u32Limit == 0xffff);
6199	Assert(u32GSAttr == 0xf3);
6200	/* 64-bit capable CPUs. */
6201	# if HC_ARCH_BITS == 64
6202	Assert(!RT_HI_U32(pCtx->cs.u64Base));
6203	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
6204	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
6205	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
6206	# endif
6207	}
6208	}
6209	#endif /* VBOX_STRICT */
6210
6211
6212	/**
6213	* Exports a guest segment register into the guest-state area in the VMCS.
6214	*
6215	* @returns VBox status code.
6216	* @param pVCpu The cross context virtual CPU structure.
6217	* @param pVmcsInfo The VMCS info. object.
6218	* @param iSegReg The segment register number (X86_SREG_XXX).
6219	* @param pSelReg Pointer to the segment selector.
6220	*
6221	* @remarks No-long-jump zone!!!
6222	*/
6223	static int hmR0VmxExportGuestSegReg(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
6224	{
6225	Assert(iSegReg < X86_SREG_COUNT);
6226	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
6227	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
6228	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
6229	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
6230
6231	uint32_t u32Access = pSelReg->Attr.u;
6232	if (pVmcsInfo->RealMode.fRealOnV86Active)
6233	{
6234	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
6235	u32Access = 0xf3;
6236	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6237	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
6238	RT_NOREF_PV(pVCpu);
6239	}
6240	else
6241	{
6242	/*
6243	* The way to differentiate between whether this is really a null selector or was just
6244	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
6245	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
6246	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
6247	* NULL selectors loaded in protected-mode have their attribute as 0.
6248	*/
6249	if (!u32Access)
6250	u32Access = X86DESCATTR_UNUSABLE;
6251	}
6252
6253	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
6254	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
6255	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
6256
6257	/*
6258	* Commit it to the VMCS.
6259	*/
6260	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel);
6261	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit);
6262	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base);
6263	rc \|= VMXWriteVmcs32(idxAttr, u32Access);
6264	AssertRCReturn(rc, rc);
6265	return rc;
6266	}
6267
6268
6269	/**
6270	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
6271	* area in the VMCS.
6272	*
6273	* @returns VBox status code.
6274	* @param pVCpu The cross context virtual CPU structure.
6275	* @param pVmxTransient The VMX-transient structure.
6276	*
6277	* @remarks Will import guest CR0 on strict builds during validation of
6278	* segments.
6279	* @remarks No-long-jump zone!!!
6280	*/
6281	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6282	{
6283	int rc = VERR_INTERNAL_ERROR_5;
6284	PVM pVM = pVCpu->CTX_SUFF(pVM);
6285	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6286	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6287
6288	/*
6289	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
6290	*/
6291	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
6292	{
6293	#ifdef VBOX_WITH_REM
6294	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6295	{
6296	Assert(!pVmxTransient->fIsNestedGuest);
6297	Assert(pVM->hm.s.vmx.pRealModeTSS);
6298	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
6299	if ( pVmcsInfo->fWasInRealMode
6300	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
6301	{
6302	/*
6303	* Notify the recompiler must flush its code-cache as the guest -may-
6304	* rewrite code it in real-mode (e.g. OpenBSD 4.0).
6305	*/
6306	REMFlushTBs(pVM);
6307	Log4Func(("Switch to protected mode detected!\n"));
6308	pVmcsInfo->fWasInRealMode = false;
6309	}
6310	}
6311	#endif
6312	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
6313	{
6314	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
6315	if (pVmcsInfo->RealMode.fRealOnV86Active)
6316	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
6317	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
6318	AssertRCReturn(rc, rc);
6319	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
6320	}
6321
6322	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
6323	{
6324	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
6325	if (pVmcsInfo->RealMode.fRealOnV86Active)
6326	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
6327	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
6328	AssertRCReturn(rc, rc);
6329	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
6330	}
6331
6332	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
6333	{
6334	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
6335	if (pVmcsInfo->RealMode.fRealOnV86Active)
6336	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
6337	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
6338	AssertRCReturn(rc, rc);
6339	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
6340	}
6341
6342	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
6343	{
6344	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
6345	if (pVmcsInfo->RealMode.fRealOnV86Active)
6346	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
6347	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
6348	AssertRCReturn(rc, rc);
6349	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
6350	}
6351
6352	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
6353	{
6354	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
6355	if (pVmcsInfo->RealMode.fRealOnV86Active)
6356	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
6357	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
6358	AssertRCReturn(rc, rc);
6359	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
6360	}
6361
6362	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
6363	{
6364	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
6365	if (pVmcsInfo->RealMode.fRealOnV86Active)
6366	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
6367	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
6368	AssertRCReturn(rc, rc);
6369	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
6370	}
6371
6372	#ifdef VBOX_STRICT
6373	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
6374	#endif
6375	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
6376	pCtx->cs.Attr.u));
6377	}
6378
6379	/*
6380	* Guest TR.
6381	*/
6382	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
6383	{
6384	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
6385
6386	/*
6387	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
6388	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
6389	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
6390	*/
6391	uint16_t u16Sel;
6392	uint32_t u32Limit;
6393	uint64_t u64Base;
6394	uint32_t u32AccessRights;
6395	if (!pVmcsInfo->RealMode.fRealOnV86Active)
6396	{
6397	u16Sel = pCtx->tr.Sel;
6398	u32Limit = pCtx->tr.u32Limit;
6399	u64Base = pCtx->tr.u64Base;
6400	u32AccessRights = pCtx->tr.Attr.u;
6401	}
6402	else
6403	{
6404	Assert(!pVmxTransient->fIsNestedGuest);
6405	Assert(pVM->hm.s.vmx.pRealModeTSS);
6406	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
6407
6408	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
6409	RTGCPHYS GCPhys;
6410	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
6411	AssertRCReturn(rc, rc);
6412
6413	X86DESCATTR DescAttr;
6414	DescAttr.u = 0;
6415	DescAttr.n.u1Present = 1;
6416	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
6417
6418	u16Sel = 0;
6419	u32Limit = HM_VTX_TSS_SIZE;
6420	u64Base = GCPhys;
6421	u32AccessRights = DescAttr.u;
6422	}
6423
6424	/* Validate. */
6425	Assert(!(u16Sel & RT_BIT(2)));
6426	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
6427	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
6428	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
6429	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
6430	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
6431	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
6432	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
6433	Assert( (u32Limit & 0xfff) == 0xfff
6434	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
6435	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
6436	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
6437
6438	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
6439	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
6440	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
6441	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
6442	AssertRCReturn(rc, rc);
6443
6444	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
6445	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
6446	}
6447
6448	/*
6449	* Guest GDTR.
6450	*/
6451	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
6452	{
6453	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
6454
6455	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
6456	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
6457	AssertRCReturn(rc, rc);
6458
6459	/* Validate. */
6460	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6461
6462	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
6463	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
6464	}
6465
6466	/*
6467	* Guest LDTR.
6468	*/
6469	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
6470	{
6471	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
6472
6473	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
6474	uint32_t u32Access;
6475	if ( !pVmxTransient->fIsNestedGuest
6476	&& !pCtx->ldtr.Attr.u)
6477	u32Access = X86DESCATTR_UNUSABLE;
6478	else
6479	u32Access = pCtx->ldtr.Attr.u;
6480
6481	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
6482	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
6483	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
6484	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
6485	AssertRCReturn(rc, rc);
6486
6487	/* Validate. */
6488	if (!(u32Access & X86DESCATTR_UNUSABLE))
6489	{
6490	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
6491	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
6492	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
6493	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
6494	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
6495	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
6496	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
6497	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
6498	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
6499	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
6500	}
6501
6502	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
6503	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
6504	}
6505
6506	/*
6507	* Guest IDTR.
6508	*/
6509	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
6510	{
6511	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
6512
6513	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
6514	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
6515	AssertRCReturn(rc, rc);
6516
6517	/* Validate. */
6518	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6519
6520	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
6521	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
6522	}
6523
6524	return VINF_SUCCESS;
6525	}
6526
6527
6528	/**
6529	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
6530	* areas.
6531	*
6532	* These MSRs will automatically be loaded to the host CPU on every successful
6533	* VM-entry and stored from the host CPU on every successful VM-exit.
6534	*
6535	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
6536	* actual host MSR values are not- updated here for performance reasons. See
6537	* hmR0VmxExportHostMsrs().
6538	*
6539	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
6540	*
6541	* @returns VBox status code.
6542	* @param pVCpu The cross context virtual CPU structure.
6543	* @param pVmxTransient The VMX-transient structure.
6544	*
6545	* @remarks No-long-jump zone!!!
6546	*/
6547	static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6548	{
6549	AssertPtr(pVCpu);
6550	AssertPtr(pVmxTransient);
6551
6552	PVM pVM = pVCpu->CTX_SUFF(pVM);
6553	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6554
6555	/*
6556	* MSRs that we use the auto-load/store MSR area in the VMCS.
6557	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
6558	* The host MSR values are updated when it's safe in hmR0VmxLazySaveHostMsrs().
6559	*
6560	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
6561	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
6562	* emulation, nothing to do here.
6563	*/
6564	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
6565	{
6566	if ( !pVmxTransient->fIsNestedGuest
6567	&& pVM->hm.s.fAllow64BitGuests)
6568	{
6569	#if HC_ARCH_BITS == 32
6570	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_KERNEL_GS_BASE);
6571	Assert(!pVmxTransient->fIsNestedGuest);
6572
6573	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_LSTAR, pCtx->msrLSTAR, true, false);
6574	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_STAR, pCtx->msrSTAR, true, false);
6575	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_SF_MASK, pCtx->msrSFMASK, true, false);
6576	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, true, false);
6577	AssertRCReturn(rc, rc);
6578	#endif
6579	}
6580	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
6581	}
6582
6583	/*
6584	* Guest Sysenter MSRs.
6585	*/
6586	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
6587	{
6588	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
6589
6590	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
6591	{
6592	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
6593	AssertRCReturn(rc, rc);
6594	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
6595	}
6596
6597	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
6598	{
6599	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
6600	AssertRCReturn(rc, rc);
6601	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
6602	}
6603
6604	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
6605	{
6606	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
6607	AssertRCReturn(rc, rc);
6608	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
6609	}
6610	}
6611
6612	/*
6613	* Guest/host EFER MSR.
6614	*/
6615	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
6616	{
6617	/* Whether we are using the VMCS to swap the EFER MSR must have been
6618	determined earlier while exporting VM-entry/VM-exit controls. */
6619	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
6620	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
6621
6622	if (hmR0VmxShouldSwapEferMsr(pVCpu))
6623	{
6624	/*
6625	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
6626	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
6627	*/
6628	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
6629	{
6630	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
6631	AssertRCReturn(rc, rc);
6632	}
6633	else
6634	{
6635	/*
6636	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
6637	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
6638	*/
6639	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, pCtx->msrEFER,
6640	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6641	AssertRCReturn(rc, rc);
6642	}
6643	}
6644	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
6645	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
6646
6647	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
6648	}
6649
6650	/*
6651	* Other MSRs.
6652	* Speculation Control (R/W).
6653	*/
6654	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
6655	{
6656	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
6657	if (pVM->cpum.ro.GuestFeatures.fIbrs)
6658	{
6659	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
6660	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6661	AssertRCReturn(rc, rc);
6662	}
6663	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
6664	}
6665
6666	return VINF_SUCCESS;
6667	}
6668
6669
6670	/**
6671	* Selects up the appropriate function to run guest code.
6672	*
6673	* @returns VBox status code.
6674	* @param pVCpu The cross context virtual CPU structure.
6675	* @param pVmxTransient The VMX-transient structure.
6676	*
6677	* @remarks No-long-jump zone!!!
6678	*/
6679	static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6680	{
6681	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6682	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6683
6684	if (CPUMIsGuestInLongModeEx(pCtx))
6685	{
6686	#ifndef VBOX_WITH_64_BITS_GUESTS
6687	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
6688	#else
6689	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
6690	/* Guest is in long mode, use the 64-bit handler (host is 64-bit). */
6691	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
6692	#endif
6693	}
6694	else
6695	{
6696	/* Guest is not in long mode, use the 32-bit handler. */
6697	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6698	}
6699	Assert(pVmcsInfo->pfnStartVM);
6700	return VINF_SUCCESS;
6701	}
6702
6703
6704	/**
6705	* Wrapper for running the guest code in VT-x.
6706	*
6707	* @returns VBox status code, no informational status codes.
6708	* @param pVCpu The cross context virtual CPU structure.
6709	* @param pVmxTransient The VMX-transient structure.
6710	*
6711	* @remarks No-long-jump zone!!!
6712	*/
6713	DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient)
6714	{
6715	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
6716	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6717	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
6718
6719	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
6720
6721	/*
6722	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
6723	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
6724	* callee-saved and thus the need for this XMM wrapper.
6725	*
6726	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
6727	*/
6728	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6729	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
6730	PVM pVM = pVCpu->CTX_SUFF(pVM);
6731	#ifdef VBOX_WITH_KERNEL_USING_XMM
6732	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu, pVmcsInfo->pfnStartVM);
6733	#else
6734	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu);
6735	#endif
6736	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
6737	return rc;
6738	}
6739
6740
6741	/**
6742	* Reports world-switch error and dumps some useful debug info.
6743	*
6744	* @param pVCpu The cross context virtual CPU structure.
6745	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
6746	* @param pVmxTransient The VMX-transient structure (only
6747	* exitReason updated).
6748	*/
6749	static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
6750	{
6751	Assert(pVCpu);
6752	Assert(pVmxTransient);
6753	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6754
6755	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
6756	switch (rcVMRun)
6757	{
6758	case VERR_VMX_INVALID_VMXON_PTR:
6759	AssertFailed();
6760	break;
6761	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
6762	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
6763	{
6764	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
6765	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
6766	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
6767	AssertRC(rc);
6768
6769	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6770	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
6771	Cannot do it here as we may have been long preempted. */
6772
6773	#ifdef VBOX_STRICT
6774	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6775	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
6776	pVmxTransient->uExitReason));
6777	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
6778	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
6779	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
6780	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
6781	else
6782	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
6783	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
6784	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
6785
6786	/* VMX control bits. */
6787	uint32_t u32Val;
6788	uint64_t u64Val;
6789	RTHCUINTREG uHCReg;
6790	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
6791	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
6792	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
6793	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
6794	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
6795	{
6796	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
6797	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
6798	}
6799	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
6800	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
6801	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
6802	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
6803	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
6804	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
6805	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
6806	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
6807	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
6808	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
6809	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
6810	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
6811	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
6812	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
6813	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
6814	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
6815	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6816	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
6817	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6818	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
6819	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
6820	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
6821	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
6822	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
6823	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
6824	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
6825	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
6826	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
6827	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
6828	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6829	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
6830	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
6831	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
6832	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6833	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6834	{
6835	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
6836	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
6837	}
6838
6839	/* Guest bits. */
6840	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
6841	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
6842	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
6843	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
6844	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
6845	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
6846	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
6847	{
6848	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
6849	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
6850	}
6851
6852	/* Host bits. */
6853	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
6854	Log4(("Host CR0 %#RHr\n", uHCReg));
6855	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
6856	Log4(("Host CR3 %#RHr\n", uHCReg));
6857	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
6858	Log4(("Host CR4 %#RHr\n", uHCReg));
6859
6860	RTGDTR HostGdtr;
6861	PCX86DESCHC pDesc;
6862	ASMGetGDTR(&HostGdtr);
6863	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
6864	Log4(("Host CS %#08x\n", u32Val));
6865	if (u32Val < HostGdtr.cbGdt)
6866	{
6867	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6868	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
6869	}
6870
6871	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
6872	Log4(("Host DS %#08x\n", u32Val));
6873	if (u32Val < HostGdtr.cbGdt)
6874	{
6875	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6876	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
6877	}
6878
6879	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
6880	Log4(("Host ES %#08x\n", u32Val));
6881	if (u32Val < HostGdtr.cbGdt)
6882	{
6883	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6884	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
6885	}
6886
6887	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
6888	Log4(("Host FS %#08x\n", u32Val));
6889	if (u32Val < HostGdtr.cbGdt)
6890	{
6891	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6892	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
6893	}
6894
6895	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
6896	Log4(("Host GS %#08x\n", u32Val));
6897	if (u32Val < HostGdtr.cbGdt)
6898	{
6899	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6900	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
6901	}
6902
6903	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
6904	Log4(("Host SS %#08x\n", u32Val));
6905	if (u32Val < HostGdtr.cbGdt)
6906	{
6907	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6908	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
6909	}
6910
6911	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
6912	Log4(("Host TR %#08x\n", u32Val));
6913	if (u32Val < HostGdtr.cbGdt)
6914	{
6915	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6916	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
6917	}
6918
6919	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
6920	Log4(("Host TR Base %#RHv\n", uHCReg));
6921	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
6922	Log4(("Host GDTR Base %#RHv\n", uHCReg));
6923	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
6924	Log4(("Host IDTR Base %#RHv\n", uHCReg));
6925	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
6926	Log4(("Host SYSENTER CS %#08x\n", u32Val));
6927	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
6928	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
6929	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
6930	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
6931	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
6932	Log4(("Host RSP %#RHv\n", uHCReg));
6933	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
6934	Log4(("Host RIP %#RHv\n", uHCReg));
6935	# if HC_ARCH_BITS == 64
6936	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
6937	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
6938	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
6939	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
6940	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
6941	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
6942	# endif
6943	#endif /* VBOX_STRICT */
6944	break;
6945	}
6946
6947	default:
6948	/* Impossible */
6949	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
6950	break;
6951	}
6952	}
6953
6954
6955	/**
6956	* Sets up the usage of TSC-offsetting and updates the VMCS.
6957	*
6958	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
6959	* VMX-preemption timer.
6960	*
6961	* @returns VBox status code.
6962	* @param pVCpu The cross context virtual CPU structure.
6963	* @param pVmxTransient The VMX-transient structure.
6964	*
6965	* @remarks No-long-jump zone!!!
6966	*/
6967	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6968	{
6969	bool fOffsettedTsc;
6970	bool fParavirtTsc;
6971	uint64_t uTscOffset;
6972	PVM pVM = pVCpu->CTX_SUFF(pVM);
6973	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6974
6975	if (pVM->hm.s.vmx.fUsePreemptTimer)
6976	{
6977	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
6978
6979	/* Make sure the returned values have sane upper and lower boundaries. */
6980	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
6981	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
6982	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
6983	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
6984
6985	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
6986	* preemption timers here. We probably need to clamp the preemption timer,
6987	* after converting the timer value to the host. */
6988	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
6989	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
6990	AssertRC(rc);
6991	}
6992	else
6993	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
6994
6995	if (fParavirtTsc)
6996	{
6997	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
6998	information before every VM-entry, hence disable it for performance sake. */
6999	#if 0
7000	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
7001	AssertRC(rc);
7002	#endif
7003	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
7004	}
7005
7006	if ( fOffsettedTsc
7007	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
7008	{
7009	if (pVmxTransient->fIsNestedGuest)
7010	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
7011	hmR0VmxSetTscOffsetVmcs(pVCpu, pVmcsInfo, uTscOffset);
7012	hmR0VmxRemoveProcCtlsVmcs(pVCpu, pVmxTransient, VMX_PROC_CTLS_RDTSC_EXIT);
7013	}
7014	else
7015	{
7016	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
7017	hmR0VmxSetProcCtlsVmcs(pVmxTransient, VMX_PROC_CTLS_RDTSC_EXIT);
7018	}
7019	}
7020
7021
7022	/**
7023	* Gets the IEM exception flags for the specified vector and IDT vectoring /
7024	* VM-exit interruption info type.
7025	*
7026	* @returns The IEM exception flags.
7027	* @param uVector The event vector.
7028	* @param uVmxEventType The VMX event type.
7029	*
7030	* @remarks This function currently only constructs flags required for
7031	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
7032	* and CR2 aspects of an exception are not included).
7033	*/
7034	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
7035	{
7036	uint32_t fIemXcptFlags;
7037	switch (uVmxEventType)
7038	{
7039	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7040	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7041	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
7042	break;
7043
7044	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7045	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
7046	break;
7047
7048	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7049	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
7050	break;
7051
7052	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
7053	{
7054	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
7055	if (uVector == X86_XCPT_BP)
7056	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
7057	else if (uVector == X86_XCPT_OF)
7058	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
7059	else
7060	{
7061	fIemXcptFlags = 0;
7062	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
7063	}
7064	break;
7065	}
7066
7067	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7068	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
7069	break;
7070
7071	default:
7072	fIemXcptFlags = 0;
7073	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
7074	break;
7075	}
7076	return fIemXcptFlags;
7077	}
7078
7079
7080	/**
7081	* Sets an event as a pending event to be injected into the guest.
7082	*
7083	* @param pVCpu The cross context virtual CPU structure.
7084	* @param u32IntInfo The VM-entry interruption-information field.
7085	* @param cbInstr The VM-entry instruction length in bytes (for software
7086	* interrupts, exceptions and privileged software
7087	* exceptions).
7088	* @param u32ErrCode The VM-entry exception error code.
7089	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
7090	* page-fault.
7091	*/
7092	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7093	RTGCUINTPTR GCPtrFaultAddress)
7094	{
7095	Assert(!pVCpu->hm.s.Event.fPending);
7096	pVCpu->hm.s.Event.fPending = true;
7097	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
7098	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
7099	pVCpu->hm.s.Event.cbInstr = cbInstr;
7100	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
7101	}
7102
7103
7104	/**
7105	* Sets an external interrupt as pending-for-injection into the VM.
7106	*
7107	* @param pVCpu The cross context virtual CPU structure.
7108	* @param u8Interrupt The external interrupt vector.
7109	*/
7110	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPU pVCpu, uint8_t u8Interrupt)
7111	{
7112	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
7113	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
7114	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7115	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7116	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7117	}
7118
7119
7120	/**
7121	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
7122	*
7123	* @param pVCpu The cross context virtual CPU structure.
7124	*/
7125	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPU pVCpu)
7126	{
7127	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
7128	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
7129	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7130	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7131	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7132	}
7133
7134
7135	/**
7136	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
7137	*
7138	* @param pVCpu The cross context virtual CPU structure.
7139	*/
7140	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
7141	{
7142	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
7143	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7144	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7145	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7146	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7147	}
7148
7149
7150	/**
7151	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7152	*
7153	* @param pVCpu The cross context virtual CPU structure.
7154	*/
7155	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
7156	{
7157	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
7158	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7159	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7160	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7161	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7162	}
7163
7164
7165	/**
7166	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
7167	*
7168	* @param pVCpu The cross context virtual CPU structure.
7169	*/
7170	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
7171	{
7172	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
7173	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7174	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7175	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7176	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7177	}
7178
7179
7180	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7181	/**
7182	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
7183	*
7184	* @param pVCpu The cross context virtual CPU structure.
7185	* @param u32ErrCode The error code for the general-protection exception.
7186	*/
7187	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
7188	{
7189	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
7190	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7191	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7192	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7193	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
7194	}
7195
7196
7197	/**
7198	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
7199	*
7200	* @param pVCpu The cross context virtual CPU structure.
7201	* @param u32ErrCode The error code for the stack exception.
7202	*/
7203	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
7204	{
7205	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
7206	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7207	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7208	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7209	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
7210	}
7211	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7212
7213
7214	/**
7215	* Fixes up attributes for the specified segment register.
7216	*
7217	* @param pVCpu The cross context virtual CPU structure.
7218	* @param pSelReg The segment register that needs fixing.
7219	* @param idxSel The VMCS field for the corresponding segment register.
7220	*/
7221	static void hmR0VmxFixUnusableSegRegAttr(PVMCPU pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7222	{
7223	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7224
7225	/*
7226	* If VT-x marks the segment as unusable, most other bits remain undefined:
7227	* - For CS the L, D and G bits have meaning.
7228	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7229	* - For the remaining data segments no bits are defined.
7230	*
7231	* The present bit and the unusable bit has been observed to be set at the
7232	* same time (the selector was supposed to be invalid as we started executing
7233	* a V8086 interrupt in ring-0).
7234	*
7235	* What should be important for the rest of the VBox code, is that the P bit is
7236	* cleared. Some of the other VBox code recognizes the unusable bit, but
7237	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7238	* safe side here, we'll strip off P and other bits we don't care about. If
7239	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7240	*
7241	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7242	*/
7243	#ifdef VBOX_STRICT
7244	uint32_t const uAttr = pSelReg->Attr.u;
7245	#endif
7246
7247	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7248	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7249	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7250
7251	#ifdef VBOX_STRICT
7252	VMMRZCallRing3Disable(pVCpu);
7253	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7254	# ifdef DEBUG_bird
7255	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7256	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7257	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7258	# endif
7259	VMMRZCallRing3Enable(pVCpu);
7260	NOREF(uAttr);
7261	#endif
7262	RT_NOREF2(pVCpu, idxSel);
7263	}
7264
7265
7266	/**
7267	* Imports a guest segment register from the current VMCS into the guest-CPU
7268	* context.
7269	*
7270	* @returns VBox status code.
7271	* @param pVCpu The cross context virtual CPU structure.
7272	* @param iSegReg The segment register number (X86_SREG_XXX).
7273	*
7274	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7275	* do not log!
7276	*/
7277	static int hmR0VmxImportGuestSegReg(PVMCPU pVCpu, uint8_t iSegReg)
7278	{
7279	Assert(iSegReg < X86_SREG_COUNT);
7280
7281	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7282	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7283	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7284	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7285
7286	uint64_t u64Base;
7287	uint32_t u32Sel, u32Limit, u32Attr;
7288	int rc = VMXReadVmcs32(idxSel, &u32Sel);
7289	rc \|= VMXReadVmcs32(idxLimit, &u32Limit);
7290	rc \|= VMXReadVmcs32(idxAttr, &u32Attr);
7291	rc \|= VMXReadVmcsGstN(idxBase, &u64Base);
7292	if (RT_SUCCESS(rc))
7293	{
7294	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7295	pSelReg->Sel = u32Sel;
7296	pSelReg->ValidSel = u32Sel;
7297	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7298	pSelReg->u32Limit = u32Limit;
7299	pSelReg->u64Base = u64Base;
7300	pSelReg->Attr.u = u32Attr;
7301	if (u32Attr & X86DESCATTR_UNUSABLE)
7302	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7303	}
7304	return rc;
7305	}
7306
7307
7308	/**
7309	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7310	*
7311	* @returns VBox status code.
7312	* @param pVCpu The cross context virtual CPU structure.
7313	*
7314	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7315	* do not log!
7316	*/
7317	static int hmR0VmxImportGuestLdtr(PVMCPU pVCpu)
7318	{
7319	uint64_t u64Base;
7320	uint32_t u32Sel, u32Limit, u32Attr;
7321	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, &u32Sel);
7322	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit);
7323	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr);
7324	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, &u64Base);
7325	if (RT_SUCCESS(rc))
7326	{
7327	pVCpu->cpum.GstCtx.ldtr.Sel = u32Sel;
7328	pVCpu->cpum.GstCtx.ldtr.ValidSel = u32Sel;
7329	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7330	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7331	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7332	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7333	if (u32Attr & X86DESCATTR_UNUSABLE)
7334	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7335	}
7336	return rc;
7337	}
7338
7339
7340	/**
7341	* Imports the guest TR from the current VMCS into the guest-CPU context.
7342	*
7343	* @returns VBox status code.
7344	* @param pVCpu The cross context virtual CPU structure.
7345	*
7346	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7347	* do not log!
7348	*/
7349	static int hmR0VmxImportGuestTr(PVMCPU pVCpu)
7350	{
7351	uint32_t u32Sel, u32Limit, u32Attr;
7352	uint64_t u64Base;
7353	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_TR_SEL, &u32Sel);
7354	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit);
7355	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr);
7356	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_TR_BASE, &u64Base);
7357	AssertRCReturn(rc, rc);
7358
7359	pVCpu->cpum.GstCtx.tr.Sel = u32Sel;
7360	pVCpu->cpum.GstCtx.tr.ValidSel = u32Sel;
7361	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7362	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7363	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7364	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7365	/* TR is the only selector that can never be unusable. */
7366	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7367	return VINF_SUCCESS;
7368	}
7369
7370
7371	/**
7372	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7373	*
7374	* @returns VBox status code.
7375	* @param pVCpu The cross context virtual CPU structure.
7376	*
7377	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7378	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7379	* instead!!!
7380	*/
7381	static int hmR0VmxImportGuestRip(PVMCPU pVCpu)
7382	{
7383	uint64_t u64Val;
7384	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7385	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7386	{
7387	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
7388	if (RT_SUCCESS(rc))
7389	{
7390	pCtx->rip = u64Val;
7391	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7392	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7393	}
7394	return rc;
7395	}
7396	return VINF_SUCCESS;
7397	}
7398
7399
7400	/**
7401	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7402	*
7403	* @returns VBox status code.
7404	* @param pVCpu The cross context virtual CPU structure.
7405	* @param pVmcsInfo The VMCS info. object.
7406	*
7407	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7408	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7409	* instead!!!
7410	*/
7411	static int hmR0VmxImportGuestRFlags(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7412	{
7413	uint32_t u32Val;
7414	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7415	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7416	{
7417	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
7418	if (RT_SUCCESS(rc))
7419	{
7420	pCtx->eflags.u32 = u32Val;
7421
7422	/* Restore eflags for real-on-v86-mode hack. */
7423	if (pVmcsInfo->RealMode.fRealOnV86Active)
7424	{
7425	pCtx->eflags.Bits.u1VM = 0;
7426	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7427	}
7428	}
7429	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7430	return rc;
7431	}
7432	return VINF_SUCCESS;
7433	}
7434
7435
7436	/**
7437	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7438	* context.
7439	*
7440	* @returns VBox status code.
7441	* @param pVCpu The cross context virtual CPU structure.
7442	* @param pVmcsInfo The VMCS info. object.
7443	*
7444	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7445	* do not log!
7446	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7447	* instead!!!
7448	*/
7449	static int hmR0VmxImportGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7450	{
7451	uint32_t u32Val;
7452	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
7453	if (RT_SUCCESS(rc))
7454	{
7455	if (!u32Val)
7456	{
7457	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7458	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7459
7460	CPUMSetGuestNmiBlocking(pVCpu, false);
7461	}
7462	else
7463	{
7464	/*
7465	* We must import RIP here to set our EM interrupt-inhibited state.
7466	* We also import RFLAGS as our code that evaluates pending interrupts
7467	* before VM-entry requires it.
7468	*/
7469	rc = hmR0VmxImportGuestRip(pVCpu);
7470	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7471	if (RT_SUCCESS(rc))
7472	{
7473	if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS \| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7474	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7475	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7476	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7477
7478	bool const fNmiBlocking = RT_BOOL(u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7479	CPUMSetGuestNmiBlocking(pVCpu, fNmiBlocking);
7480	}
7481	}
7482	}
7483	return rc;
7484	}
7485
7486
7487	/**
7488	* Worker for VMXR0ImportStateOnDemand.
7489	*
7490	* @returns VBox status code.
7491	* @param pVCpu The cross context virtual CPU structure.
7492	* @param pVmcsInfo The VMCS info. object.
7493	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7494	*/
7495	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
7496	{
7497	#define VMXLOCAL_BREAK_RC(a_rc) \
7498	if (RT_SUCCESS(a_rc)) \
7499	{ } \
7500	else \
7501	break
7502
7503	int rc = VINF_SUCCESS;
7504	PVM pVM = pVCpu->CTX_SUFF(pVM);
7505	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7506	uint64_t u64Val;
7507	uint32_t u32Val;
7508
7509	/*
7510	* Note! This is hack to workaround a mysterious BSOD observed with release builds
7511	* on Windows 10 64-bit hosts. Profile and debug builds are not affected and
7512	* neither are other host platforms.
7513	*
7514	* Committing this temporarily as it prevents BSOD.
7515	*
7516	* Update: This is very likely a compiler optimization bug, see @bugref{9180}.
7517	*/
7518	#ifdef RT_OS_WINDOWS
7519	if (pVM == 0 \|\| pVM == (void *)(uintptr_t)-1)
7520	return VERR_HM_IPE_1;
7521	#endif
7522
7523	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
7524
7525	/*
7526	* We disable interrupts to make the updating of the state and in particular
7527	* the fExtrn modification atomic wrt to preemption hooks.
7528	*/
7529	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
7530
7531	fWhat &= pCtx->fExtrn;
7532	if (fWhat)
7533	{
7534	do
7535	{
7536	if (fWhat & CPUMCTX_EXTRN_RIP)
7537	{
7538	rc = hmR0VmxImportGuestRip(pVCpu);
7539	VMXLOCAL_BREAK_RC(rc);
7540	}
7541
7542	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
7543	{
7544	rc = hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7545	VMXLOCAL_BREAK_RC(rc);
7546	}
7547
7548	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
7549	{
7550	rc = hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
7551	VMXLOCAL_BREAK_RC(rc);
7552	}
7553
7554	if (fWhat & CPUMCTX_EXTRN_RSP)
7555	{
7556	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
7557	VMXLOCAL_BREAK_RC(rc);
7558	pCtx->rsp = u64Val;
7559	}
7560
7561	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
7562	{
7563	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
7564	if (fWhat & CPUMCTX_EXTRN_CS)
7565	{
7566	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
7567	rc \|= hmR0VmxImportGuestRip(pVCpu);
7568	if (fRealOnV86Active)
7569	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
7570	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
7571	}
7572	if (fWhat & CPUMCTX_EXTRN_SS)
7573	{
7574	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
7575	if (fRealOnV86Active)
7576	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
7577	}
7578	if (fWhat & CPUMCTX_EXTRN_DS)
7579	{
7580	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
7581	if (fRealOnV86Active)
7582	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
7583	}
7584	if (fWhat & CPUMCTX_EXTRN_ES)
7585	{
7586	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
7587	if (fRealOnV86Active)
7588	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
7589	}
7590	if (fWhat & CPUMCTX_EXTRN_FS)
7591	{
7592	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
7593	if (fRealOnV86Active)
7594	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
7595	}
7596	if (fWhat & CPUMCTX_EXTRN_GS)
7597	{
7598	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
7599	if (fRealOnV86Active)
7600	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
7601	}
7602	VMXLOCAL_BREAK_RC(rc);
7603	}
7604
7605	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
7606	{
7607	if (fWhat & CPUMCTX_EXTRN_LDTR)
7608	rc \|= hmR0VmxImportGuestLdtr(pVCpu);
7609
7610	if (fWhat & CPUMCTX_EXTRN_GDTR)
7611	{
7612	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
7613	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
7614	pCtx->gdtr.pGdt = u64Val;
7615	pCtx->gdtr.cbGdt = u32Val;
7616	}
7617
7618	/* Guest IDTR. */
7619	if (fWhat & CPUMCTX_EXTRN_IDTR)
7620	{
7621	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
7622	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
7623	pCtx->idtr.pIdt = u64Val;
7624	pCtx->idtr.cbIdt = u32Val;
7625	}
7626
7627	/* Guest TR. */
7628	if (fWhat & CPUMCTX_EXTRN_TR)
7629	{
7630	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
7631	don't need to import that one. */
7632	if (!pVmcsInfo->RealMode.fRealOnV86Active)
7633	rc \|= hmR0VmxImportGuestTr(pVCpu);
7634	}
7635	VMXLOCAL_BREAK_RC(rc);
7636	}
7637
7638	if (fWhat & CPUMCTX_EXTRN_DR7)
7639	{
7640	if (!pVCpu->hm.s.fUsingHyperDR7)
7641	{
7642	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
7643	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
7644	VMXLOCAL_BREAK_RC(rc);
7645	pCtx->dr[7] = u32Val;
7646	}
7647	}
7648
7649	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
7650	{
7651	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
7652	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
7653	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
7654	pCtx->SysEnter.cs = u32Val;
7655	VMXLOCAL_BREAK_RC(rc);
7656	}
7657
7658	#if HC_ARCH_BITS == 64
7659	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
7660	{
7661	if ( pVM->hm.s.fAllow64BitGuests
7662	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7663	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
7664	}
7665
7666	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
7667	{
7668	if ( pVM->hm.s.fAllow64BitGuests
7669	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7670	{
7671	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
7672	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
7673	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
7674	}
7675	}
7676	#endif
7677
7678	if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
7679	#if HC_ARCH_BITS == 32
7680	\|\| (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS))
7681	#endif
7682	)
7683	{
7684	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
7685	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
7686	Assert(pMsrs);
7687	Assert(cMsrs <= VMX_MISC_MAX_MSRS(pVM->hm.s.vmx.Msrs.u64Misc));
7688	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
7689	for (uint32_t i = 0; i < cMsrs; i++)
7690	{
7691	uint32_t const idMsr = pMsrs[i].u32Msr;
7692	switch (idMsr)
7693	{
7694	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
7695	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
7696	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
7697	#if HC_ARCH_BITS == 32
7698	case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsrs[i].u64Value; break;
7699	case MSR_K6_STAR: pCtx->msrSTAR = pMsrs[i].u64Value; break;
7700	case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsrs[i].u64Value; break;
7701	case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsrs[i].u64Value; break;
7702	#endif
7703	default:
7704	{
7705	pCtx->fExtrn = 0;
7706	pVCpu->hm.s.u32HMError = pMsrs->u32Msr;
7707	ASMSetFlags(fEFlags);
7708	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
7709	return VERR_HM_UNEXPECTED_LD_ST_MSR;
7710	}
7711	}
7712	}
7713	}
7714
7715	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
7716	{
7717	uint64_t u64Shadow;
7718	if (fWhat & CPUMCTX_EXTRN_CR0)
7719	{
7720	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7721	* remove when we drop 32-bit host w/ 64-bit host support, see
7722	* @bugref{9180#c39}. */
7723	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
7724	#if HC_ARCH_BITS == 32
7725	uint32_t u32Shadow;
7726	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
7727	u64Shadow = u32Shadow;
7728	#else
7729	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow);
7730	#endif
7731	VMXLOCAL_BREAK_RC(rc);
7732	u64Val = u32Val;
7733	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
7734	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7735	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7736	/*
7737	* Reapply the nested-guest's CR0 fixed bits that might have been altered while
7738	* exporting the nested-guest CR0 for executing using hardware-assisted VMX.
7739	*/
7740	if (CPUMIsGuestInVmxNonRootMode(pCtx))
7741	{
7742	u64Val \|= pCtx->hwvirt.vmx.Msrs.u64Cr0Fixed0;
7743	u64Val &= pCtx->hwvirt.vmx.Msrs.u64Cr0Fixed1;
7744	}
7745	#endif
7746	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
7747	CPUMSetGuestCR0(pVCpu, u64Val);
7748	VMMRZCallRing3Enable(pVCpu);
7749	}
7750
7751	if (fWhat & CPUMCTX_EXTRN_CR4)
7752	{
7753	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7754	* remove when we drop 32-bit host w/ 64-bit host support, see
7755	* @bugref{9180#c39}. */
7756	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
7757	#if HC_ARCH_BITS == 32
7758	uint32_t u32Shadow;
7759	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
7760	u64Shadow = u32Shadow;
7761	#else
7762	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow);
7763	#endif
7764	VMXLOCAL_BREAK_RC(rc);
7765	u64Val = u32Val;
7766	u64Val = (u64Val & ~pVmcsInfo->u64Cr4Mask)
7767	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7768	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7769	/*
7770	* Reapply the nested-guest's CR4 fixed bits that might have been altered while
7771	* exporting the nested-guest CR4 for executing using hardware-assisted VMX.
7772	*/
7773	if (CPUMIsGuestInVmxNonRootMode(pCtx))
7774	{
7775	u64Val \|= pCtx->hwvirt.vmx.Msrs.u64Cr4Fixed0;
7776	u64Val &= pCtx->hwvirt.vmx.Msrs.u64Cr4Fixed1;
7777	}
7778	#endif
7779	pCtx->cr4 = u64Val;
7780	}
7781
7782	if (fWhat & CPUMCTX_EXTRN_CR3)
7783	{
7784	/* CR0.PG bit changes are always intercepted, so it's up to date. */
7785	if ( pVM->hm.s.vmx.fUnrestrictedGuest
7786	\|\| ( pVM->hm.s.fNestedPaging
7787	&& CPUMIsGuestPagingEnabledEx(pCtx)))
7788	{
7789	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
7790	VMXLOCAL_BREAK_RC(rc);
7791	if (pCtx->cr3 != u64Val)
7792	{
7793	pCtx->cr3 = u64Val;
7794	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
7795	}
7796
7797	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
7798	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
7799	if (CPUMIsGuestInPAEModeEx(pCtx))
7800	{
7801	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
7802	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
7803	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
7804	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
7805	VMXLOCAL_BREAK_RC(rc);
7806	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
7807	}
7808	}
7809	}
7810	}
7811
7812	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7813	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
7814	{
7815	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
7816	&& !CPUMIsGuestInVmxNonRootMode(pCtx))
7817	{
7818	Assert(CPUMIsGuestInVmxRootMode(pCtx));
7819	rc = hmR0VmxCopyShadowToNstGstVmcs(pVCpu, pVmcsInfo);
7820	VMXLOCAL_BREAK_RC(rc);
7821	}
7822	}
7823	#endif
7824	} while (0);
7825
7826	if (RT_SUCCESS(rc))
7827	{
7828	/* Update fExtrn. */
7829	pCtx->fExtrn &= ~fWhat;
7830
7831	/* If everything has been imported, clear the HM keeper bit. */
7832	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
7833	{
7834	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
7835	Assert(!pCtx->fExtrn);
7836	}
7837	}
7838	}
7839	else
7840	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
7841
7842	ASMSetFlags(fEFlags);
7843
7844	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
7845
7846	if (RT_SUCCESS(rc))
7847	{ /* likely */ }
7848	else
7849	return rc;
7850
7851	/*
7852	* Honor any pending CR3 updates.
7853	*
7854	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
7855	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
7856	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
7857	*
7858	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
7859	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
7860	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
7861	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
7862	*
7863	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
7864	*/
7865	if (VMMRZCallRing3IsEnabled(pVCpu))
7866	{
7867	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7868	{
7869	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
7870	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
7871	}
7872
7873	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7874	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7875
7876	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7877	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7878	}
7879
7880	return VINF_SUCCESS;
7881	#undef VMXLOCAL_BREAK_RC
7882	}
7883
7884
7885	/**
7886	* Saves the guest state from the VMCS into the guest-CPU context.
7887	*
7888	* @returns VBox status code.
7889	* @param pVCpu The cross context virtual CPU structure.
7890	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7891	*/
7892	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
7893	{
7894	AssertPtr(pVCpu);
7895	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7896	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
7897	}
7898
7899
7900	/**
7901	* Check per-VM and per-VCPU force flag actions that require us to go back to
7902	* ring-3 for one reason or another.
7903	*
7904	* @returns Strict VBox status code (i.e. informational status codes too)
7905	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7906	* ring-3.
7907	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7908	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7909	* interrupts)
7910	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7911	* all EMTs to be in ring-3.
7912	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7913	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7914	* to the EM loop.
7915	*
7916	* @param pVCpu The cross context virtual CPU structure.
7917	* @param fStepping Whether we are single-stepping the guest using the
7918	* hypervisor debugger.
7919	*
7920	* @remarks This might cause nested-guest VM-exits, caller must check if the guest
7921	* is no longer in VMX non-root mode.
7922	*/
7923	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
7924	{
7925	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7926
7927	/*
7928	* Update pending interrupts into the APIC's IRR.
7929	*/
7930	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7931	APICUpdatePendingInterrupts(pVCpu);
7932
7933	/*
7934	* Anything pending? Should be more likely than not if we're doing a good job.
7935	*/
7936	PVM pVM = pVCpu->CTX_SUFF(pVM);
7937	if ( !fStepping
7938	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7939	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7940	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7941	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7942	return VINF_SUCCESS;
7943
7944	/* Pending PGM C3 sync. */
7945	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7946	{
7947	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7948	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
7949	VBOXSTRICTRC rcStrict = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
7950	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7951	if (rcStrict != VINF_SUCCESS)
7952	{
7953	AssertRC(VBOXSTRICTRC_VAL(rcStrict));
7954	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
7955	return rcStrict;
7956	}
7957	}
7958
7959	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7960	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
7961	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7962	{
7963	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7964	int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
7965	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
7966	return rc;
7967	}
7968
7969	/* Pending VM request packets, such as hardware interrupts. */
7970	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
7971	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
7972	{
7973	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchVmReq);
7974	Log4Func(("Pending VM request forcing us back to ring-3\n"));
7975	return VINF_EM_PENDING_REQUEST;
7976	}
7977
7978	/* Pending PGM pool flushes. */
7979	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7980	{
7981	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPgmPoolFlush);
7982	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
7983	return VINF_PGM_POOL_FLUSH_PENDING;
7984	}
7985
7986	/* Pending DMA requests. */
7987	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
7988	{
7989	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchDma);
7990	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
7991	return VINF_EM_RAW_TO_R3;
7992	}
7993
7994	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7995	/* Pending nested-guest APIC-write (has highest priority among nested-guest FFs). */
7996	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
7997	{
7998	Log4Func(("Pending nested-guest APIC-write\n"));
7999	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitApicWrite(pVCpu);
8000	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
8001	return rcStrict;
8002	}
8003	/** @todo VMCPU_FF_VMX_MTF, VMCPU_FF_VMX_PREEMPT_TIMER */
8004	#endif
8005
8006	return VINF_SUCCESS;
8007	}
8008
8009
8010	/**
8011	* Converts any TRPM trap into a pending HM event. This is typically used when
8012	* entering from ring-3 (not longjmp returns).
8013	*
8014	* @param pVCpu The cross context virtual CPU structure.
8015	*/
8016	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
8017	{
8018	Assert(TRPMHasTrap(pVCpu));
8019	Assert(!pVCpu->hm.s.Event.fPending);
8020
8021	uint8_t uVector;
8022	TRPMEVENT enmTrpmEvent;
8023	RTGCUINT uErrCode;
8024	RTGCUINTPTR GCPtrFaultAddress;
8025	uint8_t cbInstr;
8026
8027	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
8028	AssertRC(rc);
8029
8030	uint32_t u32IntInfo;
8031	u32IntInfo = uVector \| VMX_IDT_VECTORING_INFO_VALID;
8032	u32IntInfo \|= HMTrpmEventTypeToVmxEventType(uVector, enmTrpmEvent);
8033
8034	rc = TRPMResetTrap(pVCpu);
8035	AssertRC(rc);
8036	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
8037	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
8038
8039	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
8040	}
8041
8042
8043	/**
8044	* Converts the pending HM event into a TRPM trap.
8045	*
8046	* @param pVCpu The cross context virtual CPU structure.
8047	*/
8048	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
8049	{
8050	Assert(pVCpu->hm.s.Event.fPending);
8051
8052	/* If a trap was already pending, we did something wrong! */
8053	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
8054
8055	uint32_t const u32IntInfo = pVCpu->hm.s.Event.u64IntInfo;
8056	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(u32IntInfo);
8057	TRPMEVENT const enmTrapType = HMVmxEventTypeToTrpmEventType(u32IntInfo);
8058
8059	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
8060
8061	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
8062	AssertRC(rc);
8063
8064	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8065	TRPMSetErrorCode(pVCpu, pVCpu->hm.s.Event.u32ErrCode);
8066
8067	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(u32IntInfo))
8068	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
8069	else if (VMX_IDT_VECTORING_INFO_TYPE(u32IntInfo) == VMX_IDT_VECTORING_INFO_TYPE_SW_INT)
8070	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
8071
8072	/* We're now done converting the pending event. */
8073	pVCpu->hm.s.Event.fPending = false;
8074	}
8075
8076
8077	/**
8078	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
8079	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
8080	*
8081	* @param pVCpu The cross context virtual CPU structure.
8082	* @param pVmcsInfo The VMCS info. object.
8083	*/
8084	static void hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8085	{
8086	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8087	{
8088	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
8089	{
8090	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
8091	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8092	AssertRC(rc);
8093	}
8094	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
8095	}
8096
8097
8098	/**
8099	* Clears the interrupt-window exiting control in the VMCS.
8100	*
8101	* @param pVmcsInfo The VMCS info. object.
8102	*/
8103	DECLINLINE(int) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8104	{
8105	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8106	{
8107	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
8108	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8109	}
8110	return VINF_SUCCESS;
8111	}
8112
8113
8114	/**
8115	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
8116	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
8117	*
8118	* @param pVCpu The cross context virtual CPU structure.
8119	* @param pVmcsInfo The VMCS info. object.
8120	*/
8121	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8122	{
8123	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8124	{
8125	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
8126	{
8127	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8128	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8129	AssertRC(rc);
8130	Log4Func(("Setup NMI-window exiting\n"));
8131	}
8132	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
8133	}
8134
8135
8136	/**
8137	* Clears the NMI-window exiting control in the VMCS.
8138	*
8139	* @param pVmcsInfo The VMCS info. object.
8140	*/
8141	DECLINLINE(int) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8142	{
8143	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8144	{
8145	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8146	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8147	}
8148	return VINF_SUCCESS;
8149	}
8150
8151
8152	/**
8153	* Does the necessary state syncing before returning to ring-3 for any reason
8154	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
8155	*
8156	* @returns VBox status code.
8157	* @param pVCpu The cross context virtual CPU structure.
8158	* @param fImportState Whether to import the guest state from the VMCS back
8159	* to the guest-CPU context.
8160	*
8161	* @remarks No-long-jmp zone!!!
8162	*/
8163	static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
8164	{
8165	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8166	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8167
8168	RTCPUID const idCpu = RTMpCpuId();
8169	Log4Func(("HostCpuId=%u\n", idCpu));
8170
8171	/*
8172	* !!! IMPORTANT !!!
8173	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
8174	*/
8175
8176	/* Save the guest state if necessary. */
8177	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8178	if (fImportState)
8179	{
8180	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8181	AssertRCReturn(rc, rc);
8182	}
8183
8184	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
8185	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8186	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8187
8188	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
8189	#ifdef VBOX_STRICT
8190	if (CPUMIsHyperDebugStateActive(pVCpu))
8191	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
8192	#endif
8193	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8194	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
8195	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
8196
8197	#if HC_ARCH_BITS == 64
8198	/* Restore host-state bits that VT-x only restores partially. */
8199	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8200	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8201	{
8202	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
8203	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8204	}
8205	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8206	#endif
8207
8208	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8209	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8210	{
8211	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
8212	if (!fImportState)
8213	{
8214	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
8215	AssertRCReturn(rc, rc);
8216	}
8217	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8218	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
8219	}
8220	else
8221	pVCpu->hm.s.vmx.fLazyMsrs = 0;
8222
8223	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8224	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8225
8226	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8227	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8228	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8229	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8230	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8231	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8232	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8233	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8234	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitVmentry);
8235	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8236
8237	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8238
8239	/** @todo This partially defeats the purpose of having preemption hooks.
8240	* The problem is, deregistering the hooks should be moved to a place that
8241	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8242	* context.
8243	*/
8244	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8245	AssertRCReturn(rc, rc);
8246
8247	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8248	/*
8249	* A valid shadow VMCS is made active as part of VM-entry. It is necessary to
8250	* clear a shadow VMCS before allowing that VMCS to become active on another
8251	* logical processor. We may or may not be importing guest state which clears
8252	* it, so cover for it here.
8253	*
8254	* See Intel spec. 24.11.1 "Software Use of Virtual-Machine Control Structures".
8255	*/
8256	if ( pVmcsInfo->pvShadowVmcs
8257	&& pVmcsInfo->fShadowVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
8258	{
8259	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
8260	AssertRCReturn(rc, rc);
8261	}
8262
8263	/*
8264	* Flag that we need to re-import the host state if we switch to this VMCS before
8265	* executing guest or nested-guest code.
8266	*/
8267	pVmcsInfo->idHostCpu = NIL_RTCPUID;
8268	#endif
8269
8270	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8271	NOREF(idCpu);
8272	return VINF_SUCCESS;
8273	}
8274
8275
8276	/**
8277	* Leaves the VT-x session.
8278	*
8279	* @returns VBox status code.
8280	* @param pVCpu The cross context virtual CPU structure.
8281	*
8282	* @remarks No-long-jmp zone!!!
8283	*/
8284	static int hmR0VmxLeaveSession(PVMCPU pVCpu)
8285	{
8286	HM_DISABLE_PREEMPT(pVCpu);
8287	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8288	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8289	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8290
8291	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8292	and done this from the VMXR0ThreadCtxCallback(). */
8293	if (!pVCpu->hm.s.fLeaveDone)
8294	{
8295	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8296	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8297	pVCpu->hm.s.fLeaveDone = true;
8298	}
8299	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8300
8301	/*
8302	* !!! IMPORTANT !!!
8303	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
8304	*/
8305
8306	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8307	/** @todo Deregistering here means we need to VMCLEAR always
8308	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8309	* for calling VMMR0ThreadCtxHookDisable here! */
8310	VMMR0ThreadCtxHookDisable(pVCpu);
8311
8312	/* Leave HM context. This takes care of local init (term). */
8313	int rc = HMR0LeaveCpu(pVCpu);
8314
8315	HM_RESTORE_PREEMPT();
8316	return rc;
8317	}
8318
8319
8320	/**
8321	* Does the necessary state syncing before doing a longjmp to ring-3.
8322	*
8323	* @returns VBox status code.
8324	* @param pVCpu The cross context virtual CPU structure.
8325	*
8326	* @remarks No-long-jmp zone!!!
8327	*/
8328	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
8329	{
8330	return hmR0VmxLeaveSession(pVCpu);
8331	}
8332
8333
8334	/**
8335	* Take necessary actions before going back to ring-3.
8336	*
8337	* An action requires us to go back to ring-3. This function does the necessary
8338	* steps before we can safely return to ring-3. This is not the same as longjmps
8339	* to ring-3, this is voluntary and prepares the guest so it may continue
8340	* executing outside HM (recompiler/IEM).
8341	*
8342	* @returns VBox status code.
8343	* @param pVCpu The cross context virtual CPU structure.
8344	* @param rcExit The reason for exiting to ring-3. Can be
8345	* VINF_VMM_UNKNOWN_RING3_CALL.
8346	*/
8347	static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
8348	{
8349	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8350
8351	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8352	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8353	{
8354	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8355	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8356	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8357	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8358	}
8359
8360	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8361	VMMRZCallRing3Disable(pVCpu);
8362	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8363
8364	/*
8365	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8366	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8367	*
8368	* This is because execution may continue from ring-3 and we would need to inject
8369	* the event from there (hence place it back in TRPM).
8370	*/
8371	if (pVCpu->hm.s.Event.fPending)
8372	{
8373	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8374	Assert(!pVCpu->hm.s.Event.fPending);
8375
8376	/* Clear the events from the VMCS. */
8377	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8378	AssertRCReturn(rc, rc);
8379	}
8380	#ifdef VBOX_STRICT
8381	else
8382	{
8383	/*
8384	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8385	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8386	* occasionally, see @bugref{9180#c42}.
8387	*
8388	* However, if the VM-entry failed, any VM entry-interruption info. field would
8389	* be left unmodified as the event would not have been injected to the guest. In
8390	* such cases, don't assert, we're not going to continue guest execution anyway.
8391	*/
8392	uint32_t uExitReason;
8393	uint32_t uEntryIntInfo;
8394	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8395	rc \|= VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8396	AssertRC(rc);
8397	Assert(VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason) \|\| !VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
8398	}
8399	#endif
8400
8401	/*
8402	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8403	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8404	* (e.g. TPR below threshold).
8405	*/
8406	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8407	{
8408	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8409	rc \|= hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8410	AssertRCReturn(rc, rc);
8411	}
8412
8413	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8414	and if we're injecting an event we should have a TRPM trap pending. */
8415	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8416	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8417	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8418	#endif
8419
8420	/* Save guest state and restore host state bits. */
8421	int rc = hmR0VmxLeaveSession(pVCpu);
8422	AssertRCReturn(rc, rc);
8423	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8424
8425	/* Thread-context hooks are unregistered at this point!!! */
8426
8427	/* Sync recompiler state. */
8428	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
8429	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
8430	\| CPUM_CHANGED_LDTR
8431	\| CPUM_CHANGED_GDTR
8432	\| CPUM_CHANGED_IDTR
8433	\| CPUM_CHANGED_TR
8434	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
8435	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
8436	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
8437	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
8438
8439	Assert(!pVCpu->hm.s.fClearTrapFlag);
8440
8441	/* Update the exit-to-ring 3 reason. */
8442	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
8443
8444	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
8445	if ( rcExit != VINF_EM_RAW_INTERRUPT
8446	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8447	{
8448	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
8449	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8450	}
8451
8452	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
8453
8454	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
8455	VMMRZCallRing3RemoveNotification(pVCpu);
8456	VMMRZCallRing3Enable(pVCpu);
8457
8458	return rc;
8459	}
8460
8461
8462	/**
8463	* VMMRZCallRing3() callback wrapper which saves the guest state before we
8464	* longjump to ring-3 and possibly get preempted.
8465	*
8466	* @returns VBox status code.
8467	* @param pVCpu The cross context virtual CPU structure.
8468	* @param enmOperation The operation causing the ring-3 longjump.
8469	* @param pvUser User argument, currently unused, NULL.
8470	*/
8471	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
8472	{
8473	RT_NOREF(pvUser);
8474	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
8475	{
8476	/*
8477	* !!! IMPORTANT !!!
8478	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
8479	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
8480	*/
8481	VMMRZCallRing3RemoveNotification(pVCpu);
8482	VMMRZCallRing3Disable(pVCpu);
8483	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
8484	RTThreadPreemptDisable(&PreemptState);
8485
8486	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8487	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8488	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8489	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8490
8491	#if HC_ARCH_BITS == 64
8492	/* Restore host-state bits that VT-x only restores partially. */
8493	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8494	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8495	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8496	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8497	#endif
8498
8499	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8500	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8501	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8502
8503	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8504	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8505	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8506
8507	/* Clear the current VMCS data back to memory (shadow VMCS if any would have been
8508	cleared as part of importing the guest state above. */
8509	hmR0VmxClearVmcs(pVmcsInfo);
8510
8511	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
8512	VMMR0ThreadCtxHookDisable(pVCpu);
8513	HMR0LeaveCpu(pVCpu);
8514	RTThreadPreemptRestore(&PreemptState);
8515	return VINF_SUCCESS;
8516	}
8517
8518	Assert(pVCpu);
8519	Assert(pvUser);
8520	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8521	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8522
8523	VMMRZCallRing3Disable(pVCpu);
8524	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8525
8526	Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
8527
8528	int rc = hmR0VmxLongJmpToRing3(pVCpu);
8529	AssertRCReturn(rc, rc);
8530
8531	VMMRZCallRing3Enable(pVCpu);
8532	return VINF_SUCCESS;
8533	}
8534
8535
8536	/**
8537	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8538	* stack.
8539	*
8540	* @returns Strict VBox status code (i.e. informational status codes too).
8541	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8542	* @param pVCpu The cross context virtual CPU structure.
8543	* @param uValue The value to push to the guest stack.
8544	*/
8545	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
8546	{
8547	/*
8548	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8549	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8550	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8551	*/
8552	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8553	if (pCtx->sp == 1)
8554	return VINF_EM_RESET;
8555	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8556	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
8557	AssertRC(rc);
8558	return rc;
8559	}
8560
8561
8562	/**
8563	* Injects an event into the guest upon VM-entry by updating the relevant fields
8564	* in the VM-entry area in the VMCS.
8565	*
8566	* @returns Strict VBox status code (i.e. informational status codes too).
8567	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8568	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8569	*
8570	* @param pVCpu The cross context virtual CPU structure.
8571	* @param pVmxTransient The VMX-transient structure.
8572	* @param pEvent The event being injected.
8573	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state. This
8574	* will be updated if necessary. This cannot not be NULL.
8575	* @param fStepping Whether we're single-stepping guest execution and should
8576	* return VINF_EM_DBG_STEPPED if the event is injected
8577	* directly (registers modified by us, not by hardware on
8578	* VM-entry).
8579	*/
8580	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
8581	uint32_t *pfIntrState)
8582	{
8583	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8584	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
8585	Assert(pfIntrState);
8586
8587	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8588	uint32_t u32IntInfo = pEvent->u64IntInfo;
8589	uint32_t const u32ErrCode = pEvent->u32ErrCode;
8590	uint32_t const cbInstr = pEvent->cbInstr;
8591	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
8592	uint8_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
8593	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
8594
8595	#ifdef VBOX_STRICT
8596	/*
8597	* Validate the error-code-valid bit for hardware exceptions.
8598	* No error codes for exceptions in real-mode.
8599	*
8600	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8601	*/
8602	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8603	&& !CPUMIsGuestInRealModeEx(pCtx))
8604	{
8605	switch (uVector)
8606	{
8607	case X86_XCPT_PF:
8608	case X86_XCPT_DF:
8609	case X86_XCPT_TS:
8610	case X86_XCPT_NP:
8611	case X86_XCPT_SS:
8612	case X86_XCPT_GP:
8613	case X86_XCPT_AC:
8614	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
8615	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8616	RT_FALL_THRU();
8617	default:
8618	break;
8619	}
8620	}
8621
8622	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8623	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
8624	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
8625	#endif
8626
8627	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8628
8629	/*
8630	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
8631	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
8632	* interrupt handler in the (real-mode) guest.
8633	*
8634	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
8635	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8636	*/
8637	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
8638	{
8639	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
8640	{
8641	/*
8642	* For CPUs with unrestricted guest execution enabled and with the guest
8643	* in real-mode, we must not set the deliver-error-code bit.
8644	*
8645	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8646	*/
8647	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
8648	}
8649	else
8650	{
8651	PVM pVM = pVCpu->CTX_SUFF(pVM);
8652	Assert(PDMVmmDevHeapIsEnabled(pVM));
8653	Assert(pVM->hm.s.vmx.pRealModeTSS);
8654	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
8655
8656	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
8657	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8658	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
8659	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
8660	AssertRCReturn(rc2, rc2);
8661
8662	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8663	size_t const cbIdtEntry = sizeof(X86IDTR16);
8664	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
8665	{
8666	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8667	if (uVector == X86_XCPT_DF)
8668	return VINF_EM_RESET;
8669
8670	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
8671	No error codes for exceptions in real-mode. */
8672	if (uVector == X86_XCPT_GP)
8673	{
8674	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
8675	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8676	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8677	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8678	HMEVENT EventXcptDf;
8679	RT_ZERO(EventXcptDf);
8680	EventXcptDf.u64IntInfo = uXcptDfInfo;
8681	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
8682	}
8683
8684	/*
8685	* If we're injecting an event with no valid IDT entry, inject a #GP.
8686	* No error codes for exceptions in real-mode.
8687	*
8688	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8689	*/
8690	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
8691	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8692	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8693	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8694	HMEVENT EventXcptGp;
8695	RT_ZERO(EventXcptGp);
8696	EventXcptGp.u64IntInfo = uXcptGpInfo;
8697	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
8698	}
8699
8700	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8701	uint16_t uGuestIp = pCtx->ip;
8702	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
8703	{
8704	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8705	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8706	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8707	}
8708	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
8709	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8710
8711	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8712	X86IDTR16 IdtEntry;
8713	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
8714	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8715	AssertRCReturn(rc2, rc2);
8716
8717	/* Construct the stack frame for the interrupt/exception handler. */
8718	VBOXSTRICTRC rcStrict;
8719	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
8720	if (rcStrict == VINF_SUCCESS)
8721	{
8722	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
8723	if (rcStrict == VINF_SUCCESS)
8724	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
8725	}
8726
8727	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8728	if (rcStrict == VINF_SUCCESS)
8729	{
8730	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8731	pCtx->rip = IdtEntry.offSel;
8732	pCtx->cs.Sel = IdtEntry.uSel;
8733	pCtx->cs.ValidSel = IdtEntry.uSel;
8734	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8735	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
8736	&& uVector == X86_XCPT_PF)
8737	pCtx->cr2 = GCPtrFault;
8738
8739	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
8740	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
8741	\| HM_CHANGED_GUEST_RSP);
8742
8743	/*
8744	* If we delivered a hardware exception (other than an NMI) and if there was
8745	* block-by-STI in effect, we should clear it.
8746	*/
8747	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
8748	{
8749	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
8750	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
8751	Log4Func(("Clearing inhibition due to STI\n"));
8752	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
8753	}
8754
8755	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8756	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
8757
8758	/*
8759	* The event has been truly dispatched to the guest. Mark it as no longer pending so
8760	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
8761	*/
8762	pVCpu->hm.s.Event.fPending = false;
8763
8764	/*
8765	* If we eventually support nested-guest execution without unrestricted guest execution,
8766	* we should set fInterceptEvents here.
8767	*/
8768	Assert(!pVmxTransient->fIsNestedGuest);
8769
8770	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
8771	if (fStepping)
8772	rcStrict = VINF_EM_DBG_STEPPED;
8773	}
8774	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8775	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8776	return rcStrict;
8777	}
8778	}
8779
8780	/*
8781	* Validate.
8782	*/
8783	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8784	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
8785
8786	/*
8787	* Inject the event into the VMCS.
8788	*/
8789	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8790	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8791	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8792	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8793	AssertRCReturn(rc, rc);
8794
8795	/*
8796	* Update guest CR2 if this is a page-fault.
8797	*/
8798	if (VMX_ENTRY_INT_INFO_IS_XCPT_PF(u32IntInfo))
8799	pCtx->cr2 = GCPtrFault;
8800
8801	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8802	return VINF_SUCCESS;
8803	}
8804
8805
8806	/**
8807	* Evaluates the event to be delivered to the guest and sets it as the pending
8808	* event.
8809	*
8810	* @returns Strict VBox status code (i.e. informational status codes too).
8811	* @param pVCpu The cross context virtual CPU structure.
8812	* @param pVmxTransient The VMX-transient structure.
8813	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
8814	*/
8815	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
8816	{
8817	Assert(pfIntrState);
8818	Assert(!TRPMHasTrap(pVCpu));
8819
8820	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8821	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8822	bool const fIsNestedGuest = pVmxTransient->fIsNestedGuest;
8823
8824	/*
8825	* Get the current interruptibility-state of the guest or nested-guest and
8826	* then figure out what needs to be injected.
8827	*/
8828	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmxTransient);
8829	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8830	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8831	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8832
8833	/* We don't support block-by-SMI yet.*/
8834	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI));
8835
8836	/* Block-by-STI must not be set when interrupts are disabled. */
8837	if (fBlockSti)
8838	{
8839	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
8840	Assert(pCtx->eflags.Bits.u1IF);
8841	}
8842
8843	/* Update interruptibility state to the caller. */
8844	*pfIntrState = fIntrState;
8845
8846	/*
8847	* Toggling of interrupt force-flags here is safe since we update TRPM on
8848	* premature exits to ring-3 before executing guest code, see hmR0VmxExitToRing3().
8849	* We must NOT restore these force-flags.
8850	*/
8851
8852	/** @todo SMI. SMIs take priority over NMIs. */
8853
8854	/*
8855	* Check if an NMI is pending and if the guest or nested-guest can receive them.
8856	* NMIs take priority over external interrupts.
8857	*/
8858	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
8859	{
8860	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
8861	if ( !pVCpu->hm.s.Event.fPending
8862	&& !fBlockNmi
8863	&& !fBlockSti
8864	&& !fBlockMovSS)
8865	{
8866	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8867	if ( fIsNestedGuest
8868	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
8869	return IEMExecVmxVmexitXcptNmi(pVCpu);
8870	#endif
8871	hmR0VmxSetPendingXcptNmi(pVCpu);
8872	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
8873	Log4Func(("Pending NMI\n"));
8874	}
8875	else if (!fIsNestedGuest)
8876	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
8877	/* else: for nested-guests, NMI-window exiting will be picked up when merging VMCS controls. */
8878	}
8879	/*
8880	* Check if an external interrupt (PIC/APIC) is pending and if the guest or nested-guest
8881	* can receive them. Once PDMGetInterrupt() returns a valid interrupt we -must- deliver
8882	* the interrupt. We can no longer re-request it from the APIC.
8883	*/
8884	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
8885	&& !pVCpu->hm.s.fSingleInstruction)
8886	{
8887	Assert(!DBGFIsStepping(pVCpu));
8888	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
8889	AssertRCReturn(rc, rc);
8890
8891	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
8892	if ( !pVCpu->hm.s.Event.fPending
8893	&& !fBlockInt
8894	&& !fBlockSti
8895	&& !fBlockMovSS)
8896	{
8897	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8898	if ( fIsNestedGuest
8899	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8900	&& !CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8901	{
8902	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0 /* uVector /, true / fIntPending */);
8903	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
8904	return rcStrict;
8905	}
8906	#endif
8907	uint8_t u8Interrupt;
8908	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
8909	if (RT_SUCCESS(rc))
8910	{
8911	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8912	if ( fIsNestedGuest
8913	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8914	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8915	{
8916	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
8917	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
8918	return rcStrict;
8919	}
8920	#endif
8921	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
8922	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
8923	}
8924	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
8925	{
8926	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
8927
8928	if ( !fIsNestedGuest
8929	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
8930	hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
8931	/* else: for nested-guests, TPR threshold is picked up while merging VMCS controls. */
8932
8933	/*
8934	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
8935	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
8936	* need to re-set this force-flag here.
8937	*/
8938	}
8939	else
8940	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
8941	}
8942	else if (!fIsNestedGuest)
8943	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
8944	/* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCS controls. */
8945	}
8946
8947	return VINF_SUCCESS;
8948	}
8949
8950
8951	/**
8952	* Injects any pending events into the guest if the guest is in a state to
8953	* receive them.
8954	*
8955	* @returns Strict VBox status code (i.e. informational status codes too).
8956	* @param pVCpu The cross context virtual CPU structure.
8957	* @param pVmxTransient The VMX-transient structure.
8958	* @param fIntrState The VT-x guest-interruptibility state.
8959	* @param fStepping Whether we are single-stepping the guest using the
8960	* hypervisor debugger and should return
8961	* VINF_EM_DBG_STEPPED if the event was dispatched
8962	* directly.
8963	*/
8964	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
8965	{
8966	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8967	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8968
8969	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8970	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8971
8972	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8973	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8974	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8975	Assert(!TRPMHasTrap(pVCpu));
8976
8977	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8978	if (pVCpu->hm.s.Event.fPending)
8979	{
8980	/*
8981	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
8982	* pending even while injecting an event and in this case, we want a VM-exit as soon as
8983	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
8984	*
8985	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
8986	*/
8987	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8988	#ifdef VBOX_STRICT
8989	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8990	{
8991	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
8992	Assert(!fBlockInt);
8993	Assert(!fBlockSti);
8994	Assert(!fBlockMovSS);
8995	}
8996	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
8997	{
8998	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8999	Assert(!fBlockSti);
9000	Assert(!fBlockMovSS);
9001	Assert(!fBlockNmi);
9002	}
9003	#endif
9004	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
9005	uIntType));
9006
9007	/*
9008	* Inject the event and get any changes to the guest-interruptibility state.
9009	*
9010	* The guest-interruptibility state may need to be updated if we inject the event
9011	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
9012	*/
9013	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
9014	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
9015
9016	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
9017	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
9018	else
9019	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
9020	}
9021
9022	/*
9023	* Update the guest-interruptibility state.
9024	*
9025	* This is required for the real-on-v86 software interrupt injection case above, as well as
9026	* updates to the guest state from ring-3 or IEM/REM.
9027	*/
9028	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
9029	AssertRCReturn(rc, rc);
9030
9031	/*
9032	* There's no need to clear the VM-entry interruption-information field here if we're not
9033	* injecting anything. VT-x clears the valid bit on every VM-exit.
9034	*
9035	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
9036	*/
9037
9038	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
9039	NOREF(fBlockMovSS); NOREF(fBlockSti);
9040	return rcStrict;
9041	}
9042
9043
9044	/**
9045	* Enters the VT-x session.
9046	*
9047	* @returns VBox status code.
9048	* @param pVCpu The cross context virtual CPU structure.
9049	*/
9050	VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu)
9051	{
9052	AssertPtr(pVCpu);
9053	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
9054	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9055
9056	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9057	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9058	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9059
9060	#ifdef VBOX_STRICT
9061	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
9062	RTCCUINTREG uHostCr4 = ASMGetCR4();
9063	if (!(uHostCr4 & X86_CR4_VMXE))
9064	{
9065	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
9066	return VERR_VMX_X86_CR4_VMXE_CLEARED;
9067	}
9068	#endif
9069
9070	/*
9071	* Load the appropriate VMCS as the current and active one.
9072	*/
9073	PVMXVMCSINFO pVmcsInfo;
9074	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
9075	if (!fInNestedGuestMode)
9076	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
9077	else
9078	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
9079	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
9080	if (RT_SUCCESS(rc))
9081	{
9082	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
9083	pVCpu->hm.s.fLeaveDone = false;
9084	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9085
9086	/*
9087	* Do the EMT scheduled L1D flush here if needed.
9088	*/
9089	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9090	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9091	else if (pVCpu->CTX_SUFF(pVM)->hm.s.fMdsClearOnSched)
9092	hmR0MdsClear();
9093	}
9094	return rc;
9095	}
9096
9097
9098	/**
9099	* The thread-context callback (only on platforms which support it).
9100	*
9101	* @param enmEvent The thread-context event.
9102	* @param pVCpu The cross context virtual CPU structure.
9103	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
9104	* @thread EMT(pVCpu)
9105	*/
9106	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
9107	{
9108	AssertPtr(pVCpu);
9109	RT_NOREF1(fGlobalInit);
9110
9111	switch (enmEvent)
9112	{
9113	case RTTHREADCTXEVENT_OUT:
9114	{
9115	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9116	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9117	VMCPU_ASSERT_EMT(pVCpu);
9118
9119	/* No longjmps (logger flushes, locks) in this fragile context. */
9120	VMMRZCallRing3Disable(pVCpu);
9121	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
9122
9123	/* Restore host-state (FPU, debug etc.) */
9124	if (!pVCpu->hm.s.fLeaveDone)
9125	{
9126	/*
9127	* Do -not- import the guest-state here as we might already be in the middle of importing
9128	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
9129	*/
9130	hmR0VmxLeave(pVCpu, false /* fImportState */);
9131	pVCpu->hm.s.fLeaveDone = true;
9132	}
9133
9134	/* Leave HM context, takes care of local init (term). */
9135	int rc = HMR0LeaveCpu(pVCpu);
9136	AssertRC(rc);
9137
9138	/* Restore longjmp state. */
9139	VMMRZCallRing3Enable(pVCpu);
9140	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
9141	break;
9142	}
9143
9144	case RTTHREADCTXEVENT_IN:
9145	{
9146	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9147	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9148	VMCPU_ASSERT_EMT(pVCpu);
9149
9150	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
9151	VMMRZCallRing3Disable(pVCpu);
9152	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
9153
9154	/* Initialize the bare minimum state required for HM. This takes care of
9155	initializing VT-x if necessary (onlined CPUs, local init etc.) */
9156	int rc = hmR0EnterCpu(pVCpu);
9157	AssertRC(rc);
9158	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9159	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9160
9161	/* Load the active VMCS as the current one. */
9162	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9163	rc = hmR0VmxLoadVmcs(pVmcsInfo);
9164	AssertRC(rc);
9165	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9166	pVCpu->hm.s.fLeaveDone = false;
9167
9168	/* Do the EMT scheduled L1D flush if needed. */
9169	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9170	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9171
9172	/* Restore longjmp state. */
9173	VMMRZCallRing3Enable(pVCpu);
9174	break;
9175	}
9176
9177	default:
9178	break;
9179	}
9180	}
9181
9182
9183	/**
9184	* Exports the host state into the VMCS host-state area.
9185	* Sets up the VM-exit MSR-load area.
9186	*
9187	* The CPU state will be loaded from these fields on every successful VM-exit.
9188	*
9189	* @returns VBox status code.
9190	* @param pVCpu The cross context virtual CPU structure.
9191	*
9192	* @remarks No-long-jump zone!!!
9193	*/
9194	static int hmR0VmxExportHostState(PVMCPU pVCpu)
9195	{
9196	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9197
9198	int rc = VINF_SUCCESS;
9199	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
9200	{
9201	rc = hmR0VmxExportHostControlRegs();
9202	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9203
9204	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
9205	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9206
9207	rc = hmR0VmxExportHostMsrs(pVCpu);
9208	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9209
9210	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
9211	}
9212	return rc;
9213	}
9214
9215
9216	/**
9217	* Saves the host state in the VMCS host-state.
9218	*
9219	* @returns VBox status code.
9220	* @param pVCpu The cross context virtual CPU structure.
9221	*
9222	* @remarks No-long-jump zone!!!
9223	*/
9224	VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
9225	{
9226	AssertPtr(pVCpu);
9227	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9228
9229	/*
9230	* Export the host state here while entering HM context.
9231	* When thread-context hooks are used, we might get preempted and have to re-save the host
9232	* state but most of the time we won't be, so do it here before we disable interrupts.
9233	*/
9234	return hmR0VmxExportHostState(pVCpu);
9235	}
9236
9237
9238	/**
9239	* Exports the guest state into the VMCS guest-state area.
9240	*
9241	* The will typically be done before VM-entry when the guest-CPU state and the
9242	* VMCS state may potentially be out of sync.
9243	*
9244	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9245	* VM-entry controls.
9246	* Sets up the appropriate VMX non-root function to execute guest code based on
9247	* the guest CPU mode.
9248	*
9249	* @returns VBox strict status code.
9250	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9251	* without unrestricted guest execution and the VMMDev is not presently
9252	* mapped (e.g. EFI32).
9253	*
9254	* @param pVCpu The cross context virtual CPU structure.
9255	* @param pVmxTransient The VMX-transient structure.
9256	*
9257	* @remarks No-long-jump zone!!!
9258	*/
9259	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9260	{
9261	AssertPtr(pVCpu);
9262	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9263	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9264
9265	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9266
9267	/*
9268	* Determine real-on-v86 mode.
9269	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9270	*/
9271	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9272	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9273	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9274	pVmcsInfo->RealMode. fRealOnV86Active = false;
9275	else
9276	{
9277	Assert(!pVmxTransient->fIsNestedGuest);
9278	pVmcsInfo->RealMode.fRealOnV86Active = true;
9279	}
9280
9281	/*
9282	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9283	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9284	*/
9285	/** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
9286	* hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
9287	* be a need to evaluate this everytime since I'm pretty sure we intercept
9288	* all guest paging mode changes. */
9289	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
9290	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9291
9292	rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9293	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9294
9295	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9296	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9297
9298	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9299	if (rcStrict == VINF_SUCCESS)
9300	{ /* likely */ }
9301	else
9302	{
9303	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9304	return rcStrict;
9305	}
9306
9307	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9308	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9309
9310	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9311	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9312
9313	rc = hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9314	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9315
9316	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9317	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9318
9319	rc = hmR0VmxExportGuestRip(pVCpu);
9320	rc \|= hmR0VmxExportGuestRsp(pVCpu);
9321	rc \|= hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9322	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9323
9324	rc = hmR0VmxExportGuestHwvirtState(pVCpu, pVmxTransient);
9325	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9326
9327	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9328	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9329	\| HM_CHANGED_GUEST_CR2
9330	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9331	\| HM_CHANGED_GUEST_X87
9332	\| HM_CHANGED_GUEST_SSE_AVX
9333	\| HM_CHANGED_GUEST_OTHER_XSAVE
9334	\| HM_CHANGED_GUEST_XCRx
9335	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9336	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9337	\| HM_CHANGED_GUEST_TSC_AUX
9338	\| HM_CHANGED_GUEST_OTHER_MSRS
9339	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9340
9341	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9342	return rc;
9343	}
9344
9345
9346	/**
9347	* Exports the state shared between the host and guest into the VMCS.
9348	*
9349	* @param pVCpu The cross context virtual CPU structure.
9350	* @param pVmxTransient The VMX-transient structure.
9351	*
9352	* @remarks No-long-jump zone!!!
9353	*/
9354	static void hmR0VmxExportSharedState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9355	{
9356	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9357	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9358
9359	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9360	{
9361	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9362	AssertRC(rc);
9363	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9364
9365	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9366	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9367	{
9368	rc = hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9369	AssertRC(rc);
9370	}
9371	}
9372
9373	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9374	{
9375	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9376	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9377	}
9378
9379	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9380	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9381	}
9382
9383
9384	/**
9385	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9386	*
9387	* @returns Strict VBox status code (i.e. informational status codes too).
9388	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9389	* without unrestricted guest execution and the VMMDev is not presently
9390	* mapped (e.g. EFI32).
9391	*
9392	* @param pVCpu The cross context virtual CPU structure.
9393	* @param pVmxTransient The VMX-transient structure.
9394	*
9395	* @remarks No-long-jump zone!!!
9396	*/
9397	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9398	{
9399	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9400	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9401	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9402
9403	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9404	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9405	#endif
9406
9407	/*
9408	* For many exits it's only RIP that changes and hence try to export it first
9409	* without going through a lot of change flag checks.
9410	*/
9411	VBOXSTRICTRC rcStrict;
9412	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9413	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9414	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
9415	{
9416	rcStrict = hmR0VmxExportGuestRip(pVCpu);
9417	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9418	{ /* likely */}
9419	else
9420	AssertMsgFailedReturn(("Failed to export guest RIP! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
9421	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9422	}
9423	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9424	{
9425	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9426	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9427	{ /* likely */}
9428	else
9429	{
9430	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9431	VBOXSTRICTRC_VAL(rcStrict)));
9432	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9433	return rcStrict;
9434	}
9435	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9436	}
9437	else
9438	rcStrict = VINF_SUCCESS;
9439
9440	#ifdef VBOX_STRICT
9441	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9442	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9443	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9444	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
9445	("fCtxChanged=%#RX64\n", fCtxChanged));
9446	#endif
9447	return rcStrict;
9448	}
9449
9450
9451	/**
9452	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9453	* and update error record fields accordingly.
9454	*
9455	* @returns VMX_IGS_* error codes.
9456	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9457	* wrong with the guest state.
9458	*
9459	* @param pVCpu The cross context virtual CPU structure.
9460	* @param pVmcsInfo The VMCS info. object.
9461	*
9462	* @remarks This function assumes our cache of the VMCS controls
9463	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9464	*/
9465	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
9466	{
9467	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9468	#define HMVMX_CHECK_BREAK(expr, err) do { \
9469	if (!(expr)) { uError = (err); break; } \
9470	} while (0)
9471
9472	int rc;
9473	PVM pVM = pVCpu->CTX_SUFF(pVM);
9474	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9475	uint32_t uError = VMX_IGS_ERROR;
9476	uint32_t u32Val;
9477	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9478
9479	do
9480	{
9481	/*
9482	* CR0.
9483	*/
9484	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
9485	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9486	uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9487	/* Exceptions for unrestricted guest execution for fixed CR0 bits (PE, PG).
9488	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9489	if (fUnrestrictedGuest)
9490	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
9491
9492	uint32_t u32GuestCr0;
9493	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
9494	AssertRCBreak(rc);
9495	HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
9496	HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
9497	if ( !fUnrestrictedGuest
9498	&& (u32GuestCr0 & X86_CR0_PG)
9499	&& !(u32GuestCr0 & X86_CR0_PE))
9500	{
9501	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9502	}
9503
9504	/*
9505	* CR4.
9506	*/
9507	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
9508	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9509	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9510
9511	uint32_t u32GuestCr4;
9512	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
9513	AssertRCBreak(rc);
9514	HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
9515	HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
9516
9517	/*
9518	* IA32_DEBUGCTL MSR.
9519	*/
9520	uint64_t u64Val;
9521	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9522	AssertRCBreak(rc);
9523	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9524	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9525	{
9526	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9527	}
9528	uint64_t u64DebugCtlMsr = u64Val;
9529
9530	#ifdef VBOX_STRICT
9531	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9532	AssertRCBreak(rc);
9533	Assert(u32Val == pVmcsInfo->u32EntryCtls);
9534	#endif
9535	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
9536
9537	/*
9538	* RIP and RFLAGS.
9539	*/
9540	uint32_t u32Eflags;
9541	#if HC_ARCH_BITS == 64
9542	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9543	AssertRCBreak(rc);
9544	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9545	if ( !fLongModeGuest
9546	\|\| !pCtx->cs.Attr.n.u1Long)
9547	{
9548	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9549	}
9550	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9551	* must be identical if the "IA-32e mode guest" VM-entry
9552	* control is 1 and CS.L is 1. No check applies if the
9553	* CPU supports 64 linear-address bits. */
9554
9555	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9556	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9557	AssertRCBreak(rc);
9558	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9559	VMX_IGS_RFLAGS_RESERVED);
9560	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9561	u32Eflags = u64Val;
9562	#else
9563	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9564	AssertRCBreak(rc);
9565	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9566	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9567	#endif
9568
9569	if ( fLongModeGuest
9570	\|\| ( fUnrestrictedGuest
9571	&& !(u32GuestCr0 & X86_CR0_PE)))
9572	{
9573	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9574	}
9575
9576	uint32_t u32EntryInfo;
9577	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9578	AssertRCBreak(rc);
9579	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
9580	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9581
9582	/*
9583	* 64-bit checks.
9584	*/
9585	#if HC_ARCH_BITS == 64
9586	if (fLongModeGuest)
9587	{
9588	HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9589	HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9590	}
9591
9592	if ( !fLongModeGuest
9593	&& (u32GuestCr4 & X86_CR4_PCIDE))
9594	{
9595	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9596	}
9597
9598	/** @todo CR3 field must be such that bits 63:52 and bits in the range
9599	* 51:32 beyond the processor's physical-address width are 0. */
9600
9601	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9602	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9603	{
9604	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9605	}
9606
9607	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9608	AssertRCBreak(rc);
9609	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9610
9611	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9612	AssertRCBreak(rc);
9613	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9614	#endif
9615
9616	/*
9617	* PERF_GLOBAL MSR.
9618	*/
9619	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
9620	{
9621	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9622	AssertRCBreak(rc);
9623	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9624	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9625	}
9626
9627	/*
9628	* PAT MSR.
9629	*/
9630	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
9631	{
9632	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9633	AssertRCBreak(rc);
9634	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9635	for (unsigned i = 0; i < 8; i++)
9636	{
9637	uint8_t u8Val = (u64Val & 0xff);
9638	if ( u8Val != 0 /* UC */
9639	&& u8Val != 1 /* WC */
9640	&& u8Val != 4 /* WT */
9641	&& u8Val != 5 /* WP */
9642	&& u8Val != 6 /* WB */
9643	&& u8Val != 7 /* UC- */)
9644	{
9645	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9646	}
9647	u64Val >>= 8;
9648	}
9649	}
9650
9651	/*
9652	* EFER MSR.
9653	*/
9654	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
9655	{
9656	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9657	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9658	AssertRCBreak(rc);
9659	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9660	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9661	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
9662	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
9663	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9664	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
9665	* iemVmxVmentryCheckGuestState(). */
9666	HMVMX_CHECK_BREAK( fUnrestrictedGuest
9667	\|\| !(u32GuestCr0 & X86_CR0_PG)
9668	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9669	VMX_IGS_EFER_LMA_LME_MISMATCH);
9670	}
9671
9672	/*
9673	* Segment registers.
9674	*/
9675	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9676	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9677	if (!(u32Eflags & X86_EFL_VM))
9678	{
9679	/* CS */
9680	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9681	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9682	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9683	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9684	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9685	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9686	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9687	/* CS cannot be loaded with NULL in protected mode. */
9688	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9689	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9690	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
9691	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9692	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
9693	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9694	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9695	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9696	else
9697	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9698
9699	/* SS */
9700	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9701	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9702	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9703	if ( !(pCtx->cr0 & X86_CR0_PE)
9704	\|\| pCtx->cs.Attr.n.u4Type == 3)
9705	{
9706	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9707	}
9708	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9709	{
9710	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9711	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9712	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9713	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9714	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9715	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9716	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9717	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9718	}
9719
9720	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
9721	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9722	{
9723	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9724	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9725	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9726	\|\| pCtx->ds.Attr.n.u4Type > 11
9727	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9728	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9729	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9730	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9731	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9732	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9733	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9734	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9735	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9736	}
9737	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9738	{
9739	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9740	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9741	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9742	\|\| pCtx->es.Attr.n.u4Type > 11
9743	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9744	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9745	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9746	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9747	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9748	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9749	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9750	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9751	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9752	}
9753	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9754	{
9755	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9756	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9757	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9758	\|\| pCtx->fs.Attr.n.u4Type > 11
9759	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9760	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9761	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9762	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9763	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9764	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9765	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9766	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9767	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9768	}
9769	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9770	{
9771	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9772	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9773	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9774	\|\| pCtx->gs.Attr.n.u4Type > 11
9775	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9776	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9777	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9778	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9779	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9780	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9781	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9782	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9783	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9784	}
9785	/* 64-bit capable CPUs. */
9786	#if HC_ARCH_BITS == 64
9787	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9788	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9789	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9790	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9791	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9792	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9793	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9794	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9795	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9796	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9797	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9798	#endif
9799	}
9800	else
9801	{
9802	/* V86 mode checks. */
9803	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9804	if (pVmcsInfo->RealMode.fRealOnV86Active)
9805	{
9806	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9807	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9808	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9809	}
9810	else
9811	{
9812	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9813	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9814	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9815	}
9816
9817	/* CS */
9818	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9819	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9820	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9821	/* SS */
9822	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9823	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9824	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9825	/* DS */
9826	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9827	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9828	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9829	/* ES */
9830	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9831	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9832	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9833	/* FS */
9834	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9835	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9836	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9837	/* GS */
9838	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9839	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9840	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9841	/* 64-bit capable CPUs. */
9842	#if HC_ARCH_BITS == 64
9843	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9844	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9845	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9846	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9847	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9848	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9849	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9850	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9851	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9852	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9853	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9854	#endif
9855	}
9856
9857	/*
9858	* TR.
9859	*/
9860	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9861	/* 64-bit capable CPUs. */
9862	#if HC_ARCH_BITS == 64
9863	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9864	#endif
9865	if (fLongModeGuest)
9866	{
9867	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9868	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9869	}
9870	else
9871	{
9872	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9873	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9874	VMX_IGS_TR_ATTR_TYPE_INVALID);
9875	}
9876	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9877	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9878	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9879	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9880	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9881	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9882	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9883	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9884
9885	/*
9886	* GDTR and IDTR.
9887	*/
9888	#if HC_ARCH_BITS == 64
9889	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9890	AssertRCBreak(rc);
9891	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9892
9893	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9894	AssertRCBreak(rc);
9895	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9896	#endif
9897
9898	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9899	AssertRCBreak(rc);
9900	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9901
9902	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9903	AssertRCBreak(rc);
9904	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9905
9906	/*
9907	* Guest Non-Register State.
9908	*/
9909	/* Activity State. */
9910	uint32_t u32ActivityState;
9911	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9912	AssertRCBreak(rc);
9913	HMVMX_CHECK_BREAK( !u32ActivityState
9914	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
9915	VMX_IGS_ACTIVITY_STATE_INVALID);
9916	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9917	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9918	uint32_t u32IntrState;
9919	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
9920	AssertRCBreak(rc);
9921	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
9922	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9923	{
9924	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9925	}
9926
9927	/** @todo Activity state and injecting interrupts. Left as a todo since we
9928	* currently don't use activity states but ACTIVE. */
9929
9930	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9931	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9932
9933	/* Guest interruptibility-state. */
9934	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9935	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
9936	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9937	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9938	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9939	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9940	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9941	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
9942	{
9943	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9944	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9945	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9946	}
9947	else if (VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
9948	{
9949	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9950	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9951	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9952	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9953	}
9954	/** @todo Assumes the processor is not in SMM. */
9955	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9956	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9957	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9958	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9959	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9960	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
9961	&& VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
9962	{
9963	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
9964	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9965	}
9966
9967	/* Pending debug exceptions. */
9968	#if HC_ARCH_BITS == 64
9969	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
9970	AssertRCBreak(rc);
9971	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9972	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9973	u32Val = u64Val; /* For pending debug exceptions checks below. */
9974	#else
9975	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
9976	AssertRCBreak(rc);
9977	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9978	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9979	#endif
9980
9981	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9982	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
9983	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9984	{
9985	if ( (u32Eflags & X86_EFL_TF)
9986	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9987	{
9988	/* Bit 14 is PendingDebug.BS. */
9989	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9990	}
9991	if ( !(u32Eflags & X86_EFL_TF)
9992	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9993	{
9994	/* Bit 14 is PendingDebug.BS. */
9995	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9996	}
9997	}
9998
9999	/* VMCS link pointer. */
10000	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10001	AssertRCBreak(rc);
10002	if (u64Val != UINT64_C(0xffffffffffffffff))
10003	{
10004	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10005	/** @todo Bits beyond the processor's physical-address width MBZ. */
10006	/** @todo SMM checks. */
10007	Assert(pVmcsInfo->HCPhysShadowVmcs == u64Val);
10008	Assert(pVmcsInfo->pvShadowVmcs);
10009	VMXVMCSREVID VmcsRevId;
10010	VmcsRevId.u = (uint32_t )pVmcsInfo->pvShadowVmcs;
10011	HMVMX_CHECK_BREAK(VmcsRevId.n.u31RevisionId == RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID),
10012	VMX_IGS_VMCS_LINK_PTR_SHADOW_VMCS_ID_INVALID);
10013	HMVMX_CHECK_BREAK(VmcsRevId.n.fIsShadowVmcs == (uint32_t)!!(pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING),
10014	VMX_IGS_VMCS_LINK_PTR_NOT_SHADOW);
10015	}
10016
10017	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10018	* not using nested paging? */
10019	if ( pVM->hm.s.fNestedPaging
10020	&& !fLongModeGuest
10021	&& CPUMIsGuestInPAEModeEx(pCtx))
10022	{
10023	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10024	AssertRCBreak(rc);
10025	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10026
10027	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10028	AssertRCBreak(rc);
10029	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10030
10031	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10032	AssertRCBreak(rc);
10033	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10034
10035	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10036	AssertRCBreak(rc);
10037	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10038	}
10039
10040	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10041	if (uError == VMX_IGS_ERROR)
10042	uError = VMX_IGS_REASON_NOT_FOUND;
10043	} while (0);
10044
10045	pVCpu->hm.s.u32HMError = uError;
10046	return uError;
10047
10048	#undef HMVMX_ERROR_BREAK
10049	#undef HMVMX_CHECK_BREAK
10050	}
10051
10052
10053	/**
10054	* Map the APIC-access page for virtualizing APIC accesses.
10055	*
10056	* This can cause a longjumps to R3 due to the acquisition of the PGM lock. Hence,
10057	* this not done as part of exporting guest state, see @bugref{8721}.
10058	*
10059	* @returns VBox status code.
10060	* @param pVCpu The cross context virtual CPU structure.
10061	*/
10062	static int hmR0VmxMapHCApicAccessPage(PVMCPU pVCpu)
10063	{
10064	PVM pVM = pVCpu->CTX_SUFF(pVM);
10065	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
10066
10067	Assert(PDMHasApic(pVM));
10068	Assert(u64MsrApicBase);
10069
10070	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
10071	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
10072
10073	/* Unalias the existing mapping. */
10074	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
10075	AssertRCReturn(rc, rc);
10076
10077	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
10078	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
10079	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
10080	AssertRCReturn(rc, rc);
10081
10082	/* Update the per-VCPU cache of the APIC base MSR. */
10083	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
10084	return VINF_SUCCESS;
10085	}
10086
10087
10088	/**
10089	* Worker function passed to RTMpOnSpecific() that is to be called on the target
10090	* CPU.
10091	*
10092	* @param idCpu The ID for the CPU the function is called on.
10093	* @param pvUser1 Null, not used.
10094	* @param pvUser2 Null, not used.
10095	*/
10096	static DECLCALLBACK(void) hmR0DispatchHostNmi(RTCPUID idCpu, void pvUser1, void pvUser2)
10097	{
10098	RT_NOREF3(idCpu, pvUser1, pvUser2);
10099	VMXDispatchHostNmi();
10100	}
10101
10102
10103	/**
10104	* Dispatching an NMI on the host CPU that received it.
10105	*
10106	* @returns VBox status code.
10107	* @param pVCpu The cross context virtual CPU structure.
10108	* @param pVmcsInfo The VMCS info. object corresponding to the VMCS that was
10109	* executing when receiving the host NMI in VMX non-root
10110	* operation.
10111	*/
10112	static int hmR0VmxExitHostNmi(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
10113	{
10114	RTCPUID const idCpu = pVmcsInfo->idHostCpu;
10115
10116	/*
10117	* We don't want to delay dispatching the NMI any more than we have to. However,
10118	* we have already chosen -not- to dispatch NMIs when interrupts were still disabled
10119	* after executing guest or nested-guest code for the following reasons:
10120	*
10121	* - We would need to perform VMREADs with interrupts disabled and is orders of
10122	* magnitude worse when we run as a guest hypervisor without VMCS shadowing
10123	* supported by the host hypervisor.
10124	*
10125	* - It affects the common VM-exit scenario and keeps interrupts disabled for a
10126	* longer period of time just for handling an edge case like host NMIs which do
10127	* not occur nearly as frequently as other VM-exits.
10128	*
10129	* Let's cover the most likely scenario first. Check if we are on the target CPU
10130	* and dispatch the NMI right away. This should be much faster than calling into
10131	* RTMpOnSpecific() machinery.
10132	*/
10133	bool fDispatched = false;
10134	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
10135	if (idCpu == RTMpCpuId())
10136	{
10137	VMXDispatchHostNmi();
10138	fDispatched = true;
10139	}
10140	ASMSetFlags(fEFlags);
10141	if (fDispatched)
10142	{
10143	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
10144	return VINF_SUCCESS;
10145	}
10146
10147	/*
10148	* RTMpOnSpecific() waits until the worker function has run on the target CPU. So
10149	* there should be no race or recursion even if we are unlucky enough to be preempted
10150	* (to the target CPU) without dispatching the host NMI above.
10151	*/
10152	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGCIpi);
10153	return RTMpOnSpecific(idCpu, &hmR0DispatchHostNmi, NULL /* pvUser1 /, NULL / pvUser2 */);
10154	}
10155
10156
10157	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10158	/**
10159	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
10160	* nested-guest using hardware-assisted VMX.
10161	*
10162	* @param pVCpu The cross context virtual CPU structure.
10163	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
10164	* @param pVmcsInfoGst The guest VMCS info. object.
10165	*/
10166	static void hmR0VmxMergeMsrBitmapNested(PCVMCPU pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
10167	{
10168	uint32_t const cbMsrBitmap = X86_PAGE_4K_SIZE;
10169	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
10170	Assert(pu64MsrBitmap);
10171
10172	/*
10173	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
10174	* MSR that is intercepted by the guest is also intercepted while executing the
10175	* nested-guest using hardware-assisted VMX.
10176	*
10177	* Note! If the nested-guest is not using an MSR bitmap, ever MSR must cause a
10178	* nested-guest VM-exit even if the outer guest is not intercepting some
10179	* MSRs. We cannot assume the caller has initialized the nested-guest
10180	* MSR bitmap in this case.
10181	*
10182	* The guest hypervisor may also switch whether it uses MSR bitmaps for
10183	* each VM-entry, hence initializing it once per-VM while setting up the
10184	* nested-guest VMCS is not sufficient.
10185	*/
10186	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10187	if (pVmcsNstGst->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10188	{
10189	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
10190	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
10191	Assert(pu64MsrBitmapNstGst);
10192	Assert(pu64MsrBitmapGst);
10193
10194	uint32_t const cFrags = cbMsrBitmap / sizeof(uint64_t);
10195	for (uint32_t i = 0; i < cFrags; i++)
10196	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
10197	}
10198	else
10199	ASMMemFill32(pu64MsrBitmap, cbMsrBitmap, UINT32_C(0xffffffff));
10200	}
10201
10202
10203	/**
10204	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
10205	* hardware-assisted VMX execution of the nested-guest.
10206	*
10207	* For a guest, we don't modify these controls once we set up the VMCS and hence
10208	* this function is never called.
10209	*
10210	* For nested-guests since the guest hypervisor provides these controls on every
10211	* nested-guest VM-entry and could potentially change them everytime we need to
10212	* merge them before every nested-guest VM-entry.
10213	*
10214	* @returns VBox status code.
10215	* @param pVCpu The cross context virtual CPU structure.
10216	*/
10217	static int hmR0VmxMergeVmcsNested(PVMCPU pVCpu)
10218	{
10219	PVM pVM = pVCpu->CTX_SUFF(pVM);
10220	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
10221	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10222	Assert(pVmcsNstGst);
10223
10224	/*
10225	* Merge the controls with the requirements of the guest VMCS.
10226	*
10227	* We do not need to validate the nested-guest VMX features specified in the nested-guest
10228	* VMCS with the features supported by the physical CPU as it's already done by the
10229	* VMLAUNCH/VMRESUME instruction emulation.
10230	*
10231	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the guest are
10232	* derived from the VMX features supported by the physical CPU.
10233	*/
10234
10235	/* Pin-based VM-execution controls. */
10236	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
10237
10238	/* Processor-based VM-execution controls. */
10239	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
10240	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
10241	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
10242	\| VMX_PROC_CTLS_USE_TPR_SHADOW
10243	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
10244
10245	/* Secondary processor-based VM-execution controls. */
10246	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
10247	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
10248	\| VMX_PROC_CTLS2_INVPCID
10249	\| VMX_PROC_CTLS2_VMCS_SHADOWING
10250	\| VMX_PROC_CTLS2_RDTSCP
10251	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
10252	\| VMX_PROC_CTLS2_APIC_REG_VIRT
10253	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
10254	\| VMX_PROC_CTLS2_VMFUNC));
10255
10256	/*
10257	* VM-entry controls:
10258	* These controls contains state that depends on the nested-guest state (primarily
10259	* EFER MSR) and is thus not constant between VMLAUNCH/VMRESUME and the nested-guest
10260	* VM-exit. Although the guest hypervisor cannot change it, we need to in order to
10261	* properly continue executing the nested-guest if the EFER MSR changes but does not
10262	* cause a nested-guest VM-exits.
10263	*
10264	* VM-exit controls:
10265	* These controls specify the host state on return. We cannot use the controls from
10266	* the guest hypervisor state as is as it would contain the guest state rather than
10267	* the host state. Since the host state is subject to change (e.g. preemption, trips
10268	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
10269	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
10270	*
10271	* VM-entry MSR-load:
10272	* The guest MSRs from the VM-entry MSR-load area are already loaded into the guest-CPU
10273	* context by the VMLAUNCH/VMRESUME instruction emulation.
10274	*
10275	* VM-exit MSR-store:
10276	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU context
10277	* back into the VM-exit MSR-store area.
10278	*
10279	* VM-exit MSR-load areas:
10280	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence, we
10281	* can entirely ignore what the guest hypervisor wants to load here.
10282	*/
10283
10284	/*
10285	* Exception bitmap.
10286	*
10287	* We could remove #UD from the guest bitmap and merge it with the nested-guest bitmap
10288	* here (and avoid doing anything while exporting nested-guest state), but to keep the
10289	* code more flexible if intercepting exceptions become more dynamic in the future we do
10290	* it as part of exporting the nested-guest state.
10291	*/
10292	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
10293
10294	/*
10295	* CR0/CR4 guest/host mask.
10296	*
10297	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest must
10298	* cause VM-exits, so we need to merge them here.
10299	*/
10300	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
10301	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
10302
10303	/*
10304	* Page-fault error-code mask and match.
10305	*
10306	* Although we require unrestricted guest execution (and thereby nested-paging) for
10307	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
10308	* normally intercept #PFs, it might intercept them for debugging purposes.
10309	*
10310	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF filters.
10311	* If the outer guest is intercepting #PFs, we must intercept all #PFs.
10312	*/
10313	uint32_t u32XcptPFMask;
10314	uint32_t u32XcptPFMatch;
10315	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
10316	{
10317	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
10318	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
10319	}
10320	else
10321	{
10322	u32XcptPFMask = 0;
10323	u32XcptPFMatch = 0;
10324	}
10325
10326	/*
10327	* Pause-Loop exiting.
10328	*/
10329	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10330	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10331
10332	/*
10333	* I/O Bitmap.
10334	*
10335	* We do not use the I/O bitmap that may be provided by the guest hypervisor as we always
10336	* intercept all I/O port accesses.
10337	*/
10338	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10339
10340	/*
10341	* VMCS shadowing.
10342	*
10343	* We do not yet expose VMCS shadowing to the guest and thus VMCS shadowing should not be
10344	* enabled while executing the nested-guest.
10345	*/
10346	Assert(!(u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING));
10347
10348	/*
10349	* APIC-access page.
10350	*/
10351	RTHCPHYS HCPhysApicAccess;
10352	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10353	{
10354	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS);
10355	RTGCPHYS const GCPhysApicAccess = pVmcsNstGst->u64AddrApicAccess.u;
10356
10357	/** @todo NSTVMX: This is not really correct but currently is required to make
10358	* things work. We need to re-register the page handler when we fallback to
10359	* IEM execution of the nested-guest! */
10360	PGMHandlerPhysicalDeregister(pVM, GCPhysApicAccess);
10361
10362	void *pvPage;
10363	PGMPAGEMAPLOCK PgMapLockApicAccess;
10364	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysApicAccess, &pvPage, &PgMapLockApicAccess);
10365	if (RT_SUCCESS(rc))
10366	{
10367	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysApicAccess, &HCPhysApicAccess);
10368	AssertMsgRCReturn(rc, ("Failed to get host-physical address for APIC-access page at %#RGp\n", GCPhysApicAccess), rc);
10369
10370	/*
10371	* We can release the page lock here because the APIC-access page is never read or
10372	* written to but merely serves as a placeholder in the shadow/nested page tables
10373	* to cause VM-exits or re-direct the access to the virtual-APIC page.
10374	*/
10375	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &PgMapLockApicAccess);
10376	}
10377	else
10378	return rc;
10379	}
10380	else
10381	HCPhysApicAccess = 0;
10382
10383	/*
10384	* Virtual-APIC page and TPR threshold.
10385	*/
10386	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10387	RTHCPHYS HCPhysVirtApic;
10388	uint32_t u32TprThreshold;
10389	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10390	{
10391	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW);
10392
10393	void *pvPage;
10394	RTGCPHYS const GCPhysVirtApic = pVmcsNstGst->u64AddrVirtApic.u;
10395	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysVirtApic, &pvPage, &pVCpu->hm.s.vmx.PgMapLockVirtApic);
10396	AssertMsgRCReturn(rc, ("Failed to get current-context pointer for virtual-APIC page at %#RGp\n", GCPhysVirtApic), rc);
10397
10398	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysVirtApic, &HCPhysVirtApic);
10399	AssertMsgRCReturn(rc, ("Failed to get host-physical address for virtual-APIC page at %#RGp\n", GCPhysVirtApic), rc);
10400	pVCpu->hm.s.vmx.fVirtApicPageLocked = true;
10401
10402	u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10403	}
10404	else
10405	{
10406	HCPhysVirtApic = 0;
10407	u32TprThreshold = 0;
10408
10409	/*
10410	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10411	* used by the guest hypervisor. Preventing MMIO accesses to the physical APIC will
10412	* be taken care of by EPT/shadow paging.
10413	*/
10414	if (pVM->hm.s.fAllow64BitGuests)
10415	{
10416	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10417	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10418	}
10419	}
10420
10421	/*
10422	* Validate basic assumptions.
10423	*/
10424	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10425	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10426	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10427
10428	/*
10429	* Commit it to the nested-guest VMCS.
10430	*/
10431	int rc = VINF_SUCCESS;
10432	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10433	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10434	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10435	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10436	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10437	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10438	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10439	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10440	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10441	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10442	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10443	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10444	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10445	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10446	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10447	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10448	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10449	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10450	{
10451	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10452	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10453	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10454	}
10455	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10456	{
10457	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10458	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
10459	}
10460	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10461	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
10462	AssertRCReturn(rc, rc);
10463
10464	/*
10465	* Update the nested-guest VMCS cache.
10466	*/
10467	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10468	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10469	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10470	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10471	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10472	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10473	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10474	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10475	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10476
10477	/*
10478	* We need to flush the TLB if we are switching the APIC-access page address.
10479	* See Intel spec. 28.3.3.4 "Guidelines for Use of the INVEPT Instruction".
10480	*/
10481	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10482	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = true;
10483
10484	/*
10485	* MSR bitmap.
10486	*
10487	* The MSR bitmap address has already been initialized while setting up the nested-guest
10488	* VMCS, here we need to merge the MSR bitmaps.
10489	*/
10490	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10491	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10492
10493	return VINF_SUCCESS;
10494	}
10495	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10496
10497
10498	/**
10499	* Does the preparations before executing guest code in VT-x.
10500	*
10501	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10502	* recompiler/IEM. We must be cautious what we do here regarding committing
10503	* guest-state information into the VMCS assuming we assuredly execute the
10504	* guest in VT-x mode.
10505	*
10506	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10507	* the common-state (TRPM/forceflags), we must undo those changes so that the
10508	* recompiler/IEM can (and should) use them when it resumes guest execution.
10509	* Otherwise such operations must be done when we can no longer exit to ring-3.
10510	*
10511	* @returns Strict VBox status code (i.e. informational status codes too).
10512	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10513	* have been disabled.
10514	* @retval VINF_VMX_VMEXIT if a nested-guest VM-exit occurs (e.g., while evaluating
10515	* pending events).
10516	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10517	* double-fault into the guest.
10518	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10519	* dispatched directly.
10520	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10521	*
10522	* @param pVCpu The cross context virtual CPU structure.
10523	* @param pVmxTransient The VMX-transient structure.
10524	* @param fStepping Whether we are single-stepping the guest in the
10525	* hypervisor debugger. Makes us ignore some of the reasons
10526	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10527	* if event dispatching took place.
10528	*/
10529	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
10530	{
10531	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10532
10533	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
10534	if (pVmxTransient->fIsNestedGuest)
10535	{
10536	RT_NOREF2(pVCpu, fStepping);
10537	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
10538	return VINF_EM_RESCHEDULE_REM;
10539	}
10540	#endif
10541
10542	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
10543	PGMRZDynMapFlushAutoSet(pVCpu);
10544	#endif
10545
10546	/*
10547	* Check and process force flag actions, some of which might require us to go back to ring-3.
10548	*/
10549	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
10550	if (rcStrict == VINF_SUCCESS)
10551	{
10552	/* FFs don't get set all the time. */
10553	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10554	if ( pVmxTransient->fIsNestedGuest
10555	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
10556	{
10557	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
10558	return VINF_VMX_VMEXIT;
10559	}
10560	#endif
10561	}
10562	else
10563	return rcStrict;
10564
10565	/*
10566	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
10567	*/
10568	/** @todo Doing this from ring-3 after VM setup phase causes a
10569	* VERR_IOM_MMIO_RANGE_NOT_FOUND guru while booting Visa 64 SMP VM. No
10570	* idea why atm. */
10571	PVM pVM = pVCpu->CTX_SUFF(pVM);
10572	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
10573	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10574	&& PDMHasApic(pVM))
10575	{
10576	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
10577	AssertRCReturn(rc, rc);
10578	}
10579
10580	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10581	/*
10582	* Merge guest VMCS controls with the nested-guest VMCS controls.
10583	*
10584	* Even if we have not executed the guest prior to this (e.g. when resuming from a
10585	* saved state), we should be okay with merging controls as we initialize the
10586	* guest VMCS controls as part of VM setup phase.
10587	*/
10588	if ( pVmxTransient->fIsNestedGuest
10589	&& !pVCpu->hm.s.vmx.fMergedNstGstCtls)
10590	{
10591	int rc = hmR0VmxMergeVmcsNested(pVCpu);
10592	AssertRCReturn(rc, rc);
10593	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
10594	}
10595	#endif
10596
10597	/*
10598	* Evaluate events to be injected into the guest.
10599	*
10600	* Events in TRPM can be injected without inspecting the guest state.
10601	* If any new events (interrupts/NMI) are pending currently, we try to set up the
10602	* guest to cause a VM-exit the next time they are ready to receive the event.
10603	*
10604	* With nested-guests, evaluating pending events may cause VM-exits.
10605	*/
10606	if (TRPMHasTrap(pVCpu))
10607	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
10608
10609	uint32_t fIntrState;
10610	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
10611
10612	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10613	/*
10614	* While evaluating pending events if something failed (unlikely) or if we were
10615	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
10616	*/
10617	if (rcStrict != VINF_SUCCESS)
10618	return rcStrict;
10619	if ( pVmxTransient->fIsNestedGuest
10620	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
10621	{
10622	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
10623	return VINF_VMX_VMEXIT;
10624	}
10625	#else
10626	Assert(rcStrict == VINF_SUCCESS);
10627	#endif
10628
10629	/*
10630	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
10631	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
10632	* also result in triple-faulting the VM.
10633	*
10634	* With nested-guests, the above does not apply since unrestricted guest execution is a
10635	* requirement. Regardless, we do this here to avoid duplicating code elsewhere.
10636	*/
10637	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
10638	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10639	{ /* likely */ }
10640	else
10641	{
10642	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
10643	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10644	return rcStrict;
10645	}
10646
10647	/*
10648	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
10649	* import CR3 themselves. We will need to update them here, as even as late as the above
10650	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
10651	* the below force flags to be set.
10652	*/
10653	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
10654	{
10655	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
10656	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
10657	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
10658	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
10659	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10660	}
10661	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
10662	{
10663	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
10664	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10665	}
10666
10667	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10668	/* Paranoia. */
10669	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10670	#endif
10671
10672	/*
10673	* No longjmps to ring-3 from this point on!!!
10674	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
10675	* This also disables flushing of the R0-logger instance (if any).
10676	*/
10677	VMMRZCallRing3Disable(pVCpu);
10678
10679	/*
10680	* Export the guest state bits.
10681	*
10682	* We cannot perform longjmps while loading the guest state because we do not preserve the
10683	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
10684	* CPU migration.
10685	*
10686	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
10687	* registers. Hence, exporting of the guest state needs to be done -after- injection of events.
10688	*/
10689	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
10690	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10691	{ /* likely */ }
10692	else
10693	{
10694	VMMRZCallRing3Enable(pVCpu);
10695	return rcStrict;
10696	}
10697
10698	/*
10699	* We disable interrupts so that we don't miss any interrupts that would flag preemption
10700	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
10701	* preemption disabled for a while. Since this is purely to aid the
10702	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
10703	* disable interrupt on NT.
10704	*
10705	* We need to check for force-flags that could've possible been altered since we last
10706	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
10707	* see @bugref{6398}).
10708	*
10709	* We also check a couple of other force-flags as a last opportunity to get the EMT back
10710	* to ring-3 before executing guest code.
10711	*/
10712	pVmxTransient->fEFlags = ASMIntDisableFlags();
10713
10714	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
10715	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
10716	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
10717	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
10718	{
10719	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
10720	{
10721	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10722	/*
10723	* If we are executing a nested-guest make sure that we should intercept subsequent
10724	* events. The one we are injecting might be part of VM-entry. This is mainly to keep
10725	* the VM-exit instruction emulation happy.
10726	*/
10727	if (pVmxTransient->fIsNestedGuest)
10728	pVCpu->cpum.GstCtx.hwvirt.vmx.fInterceptEvents = true;
10729	#endif
10730
10731	/*
10732	* We've injected any pending events. This is really the point of no return (to ring-3).
10733	*
10734	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
10735	* returns from this function, so do -not- enable them here.
10736	*/
10737	pVCpu->hm.s.Event.fPending = false;
10738	return VINF_SUCCESS;
10739	}
10740
10741	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
10742	rcStrict = VINF_EM_RAW_INTERRUPT;
10743	}
10744	else
10745	{
10746	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
10747	rcStrict = VINF_EM_RAW_TO_R3;
10748	}
10749
10750	ASMSetFlags(pVmxTransient->fEFlags);
10751	VMMRZCallRing3Enable(pVCpu);
10752
10753	return rcStrict;
10754	}
10755
10756
10757	/**
10758	* Final preparations before executing guest code using hardware-assisted VMX.
10759	*
10760	* We can no longer get preempted to a different host CPU and there are no returns
10761	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
10762	* failures), this function is not intended to fail sans unrecoverable hardware
10763	* errors.
10764	*
10765	* @param pVCpu The cross context virtual CPU structure.
10766	* @param pVmxTransient The VMX-transient structure.
10767	*
10768	* @remarks Called with preemption disabled.
10769	* @remarks No-long-jump zone!!!
10770	*/
10771	static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10772	{
10773	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10774	Assert(VMMR0IsLogFlushDisabled(pVCpu));
10775	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
10776	Assert(!pVCpu->hm.s.Event.fPending);
10777
10778	/*
10779	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
10780	*/
10781	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10782	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
10783
10784	PVM pVM = pVCpu->CTX_SUFF(pVM);
10785	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10786	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
10787	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
10788
10789	if (!CPUMIsGuestFPUStateActive(pVCpu))
10790	{
10791	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10792	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
10793	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
10794	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10795	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
10796	}
10797
10798	/*
10799	* Re-export the host state bits as we may've been preempted (only happens when
10800	* thread-context hooks are used or when the VM start function changes) or if
10801	* the host CR0 is modified while loading the guest FPU state above.
10802	*
10803	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
10804	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
10805	* see @bugref{8432}.
10806	*
10807	* This may also happen when switching to/from a nested-guest VMCS without leaving
10808	* ring-0.
10809	*/
10810	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
10811	{
10812	hmR0VmxExportHostState(pVCpu);
10813	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportHostState);
10814	}
10815	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
10816
10817	/*
10818	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
10819	*/
10820	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
10821	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
10822	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
10823
10824	/*
10825	* Store status of the shared guest/host debug state at the time of VM-entry.
10826	*/
10827	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
10828	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
10829	{
10830	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
10831	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
10832	}
10833	else
10834	#endif
10835	{
10836	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
10837	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
10838	}
10839
10840	/*
10841	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
10842	* more than one conditional check. The post-run side of our code shall determine
10843	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
10844	*/
10845	if (pVmcsInfo->pbVirtApic)
10846	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
10847
10848	/*
10849	* Update the host MSRs values in the VM-exit MSR-load area.
10850	*/
10851	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
10852	{
10853	if (pVmcsInfo->cExitMsrLoad > 0)
10854	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
10855	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
10856	}
10857
10858	/*
10859	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
10860	* VMX-preemption timer based on the next virtual sync clock deadline.
10861	*/
10862	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
10863	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
10864	{
10865	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
10866	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
10867	}
10868
10869	/* Record statistics of how often we use TSC offsetting as opposed to intercepting RDTSC/P. */
10870	bool const fIsRdtscIntercepted = RT_BOOL(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT);
10871	if (!fIsRdtscIntercepted)
10872	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
10873	else
10874	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
10875
10876	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
10877	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
10878	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
10879	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
10880	pVmcsInfo->idHostCpu = idCurrentCpu; /* Update the CPU for which we updated host-state in this VMCS. */
10881
10882	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
10883
10884	TMNotifyStartOfExecution(pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
10885	as we're about to start executing the guest . */
10886
10887	/*
10888	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
10889	*
10890	* This is done this late as updating the TSC offsetting/preemption timer above
10891	* figures out if we can skip intercepting RDTSCP by calculating the number of
10892	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
10893	*/
10894	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
10895	&& !fIsRdtscIntercepted)
10896	{
10897	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
10898
10899	/* NB: Because we call hmR0VmxAddAutoLoadStoreMsr with fUpdateHostMsr=true,
10900	it's safe even after hmR0VmxUpdateAutoLoadHostMsrs has already been done. */
10901	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
10902	true /* fSetReadWrite /, true / fUpdateHostMsr */);
10903	AssertRC(rc);
10904	Assert(!pVmxTransient->fRemoveTscAuxMsr);
10905	pVmxTransient->fRemoveTscAuxMsr = true;
10906	}
10907
10908	#ifdef VBOX_STRICT
10909	Assert(pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs);
10910	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
10911	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
10912	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest));
10913	#endif
10914
10915	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
10916	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
10917	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs,
10918	* see @bugref{9180#c54}. */
10919	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
10920	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
10921	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
10922	#endif
10923	}
10924
10925
10926	/**
10927	* First C routine invoked after running guest code using hardware-assisted VMX.
10928	*
10929	* @param pVCpu The cross context virtual CPU structure.
10930	* @param pVmxTransient The VMX-transient structure.
10931	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
10932	*
10933	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
10934	*
10935	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
10936	* unconditionally when it is safe to do so.
10937	*/
10938	static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
10939	{
10940	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
10941
10942	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
10943	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
10944	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
10945	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
10946	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
10947	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
10948
10949	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10950	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10951	{
10952	uint64_t uGstTsc;
10953	if (!pVmxTransient->fIsNestedGuest)
10954	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10955	else
10956	{
10957	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10958	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
10959	}
10960	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
10961	}
10962
10963	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
10964	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
10965	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10966
10967	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
10968	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10969	#ifdef VBOX_STRICT
10970	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
10971	#endif
10972	Assert(!ASMIntAreEnabled());
10973	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
10974	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10975
10976	#ifdef HMVMX_ALWAYS_CLEAN_TRANSIENT
10977	/*
10978	* Clean all the VMCS fields in the transient structure before reading
10979	* anything from the VMCS.
10980	*/
10981	pVmxTransient->uExitReason = 0;
10982	pVmxTransient->uExitIntErrorCode = 0;
10983	pVmxTransient->uExitQual = 0;
10984	pVmxTransient->uGuestLinearAddr = 0;
10985	pVmxTransient->uExitIntInfo = 0;
10986	pVmxTransient->cbInstr = 0;
10987	pVmxTransient->ExitInstrInfo.u = 0;
10988	pVmxTransient->uEntryIntInfo = 0;
10989	pVmxTransient->uEntryXcptErrorCode = 0;
10990	pVmxTransient->cbEntryInstr = 0;
10991	pVmxTransient->uIdtVectoringInfo = 0;
10992	pVmxTransient->uIdtVectoringErrorCode = 0;
10993	#endif
10994
10995	/*
10996	* Save the basic VM-exit reason and check if the VM-entry failed.
10997	* See Intel spec. 24.9.1 "Basic VM-exit Information".
10998	*/
10999	uint32_t uExitReason;
11000	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
11001	AssertRC(rc);
11002	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
11003	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
11004
11005	/*
11006	* Log the VM-exit before logging anything else as otherwise it might be a
11007	* tad confusing what happens before and after the world-switch.
11008	*/
11009	HMVMX_LOG_EXIT(pVCpu, uExitReason);
11010
11011	/*
11012	* Remove the TSC_AUX MSR from the auto-load/store MSR area and reset any MSR
11013	* bitmap permissions, if it was added before VM-entry.
11014	*/
11015	if (pVmxTransient->fRemoveTscAuxMsr)
11016	{
11017	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
11018	pVmxTransient->fRemoveTscAuxMsr = false;
11019	}
11020
11021	/*
11022	* Check if VMLAUNCH/VMRESUME succeeded.
11023	* If this failed, we cause a guru meditation and cease further execution.
11024	*
11025	* However, if we are executing a nested-guest we might fail if we use the
11026	* fast path rather than fully emulating VMLAUNCH/VMRESUME instruction in IEM.
11027	*/
11028	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
11029	{
11030	/*
11031	* Update the VM-exit history array here even if the VM-entry failed due to:
11032	* - Invalid guest state.
11033	* - MSR loading.
11034	* - Machine-check event.
11035	*
11036	* In any of the above cases we will still have a "valid" VM-exit reason
11037	* despite @a fVMEntryFailed being false.
11038	*
11039	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
11040	*
11041	* Note! We don't have CS or RIP at this point. Will probably address that later
11042	* by amending the history entry added here.
11043	*/
11044	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
11045	UINT64_MAX, uHostTsc);
11046
11047	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
11048	{
11049	VMMRZCallRing3Enable(pVCpu);
11050
11051	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
11052	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
11053
11054	#ifdef HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
11055	rc = hmR0VmxReadAllRoFieldsVmcs(pVCpu, pVmxTransient);
11056	AssertRC(rc);
11057	#endif
11058	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
11059	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11060	AssertRC(rc);
11061	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
11062	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
11063	AssertRC(rc);
11064	#else
11065	/*
11066	* Import the guest-interruptibility state always as we need it while evaluating
11067	* injecting events on re-entry.
11068	*
11069	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
11070	* checking for real-mode while exporting the state because all bits that cause
11071	* mode changes wrt CR0 are intercepted.
11072	*/
11073	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
11074	AssertRC(rc);
11075	#endif
11076
11077	/*
11078	* Sync the TPR shadow with our APIC state.
11079	*/
11080	if ( !pVmxTransient->fIsNestedGuest
11081	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
11082	{
11083	Assert(pVmcsInfo->pbVirtApic);
11084	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
11085	{
11086	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
11087	AssertRC(rc);
11088	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11089	}
11090	}
11091
11092	Assert(VMMRZCallRing3IsEnabled(pVCpu));
11093	return;
11094	}
11095	}
11096	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11097	else if (pVmxTransient->fIsNestedGuest)
11098	AssertMsgFailed(("VMLAUNCH/VMRESUME failed but shouldn't happen when VMLAUNCH/VMRESUME was emulated in IEM!\n"));
11099	#endif
11100	else
11101	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
11102
11103	VMMRZCallRing3Enable(pVCpu);
11104	}
11105
11106
11107	/**
11108	* Runs the guest code using hardware-assisted VMX the normal way.
11109	*
11110	* @returns VBox status code.
11111	* @param pVCpu The cross context virtual CPU structure.
11112	* @param pcLoops Pointer to the number of executed loops.
11113	*/
11114	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, uint32_t *pcLoops)
11115	{
11116	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11117	Assert(pcLoops);
11118	Assert(*pcLoops <= cMaxResumeLoops);
11119	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11120
11121	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11122	/*
11123	* Switch to the guest VMCS as we may have transitioned from executing the nested-guest
11124	* without leaving ring-0. Otherwise, if we came from ring-3 we would have loaded the
11125	* guest VMCS while entering the VMX ring-0 session.
11126	*/
11127	if (pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
11128	{
11129	int rc = hmR0VmxSwitchToGstOrNstGstVmcs(pVCpu, false /* fSwitchToNstGstVmcs */);
11130	if (RT_SUCCESS(rc))
11131	{ /* likely */ }
11132	else
11133	{
11134	LogRelFunc(("Failed to switch to the guest VMCS. rc=%Rrc\n", rc));
11135	return rc;
11136	}
11137	}
11138	#endif
11139
11140	VMXTRANSIENT VmxTransient;
11141	RT_ZERO(VmxTransient);
11142	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11143
11144	/* Paranoia. */
11145	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
11146
11147	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11148	for (;;)
11149	{
11150	Assert(!HMR0SuspendPending());
11151	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11152	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11153
11154	/*
11155	* Preparatory work for running nested-guest code, this may force us to
11156	* return to ring-3.
11157	*
11158	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11159	*/
11160	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
11161	if (rcStrict != VINF_SUCCESS)
11162	break;
11163
11164	/* Interrupts are disabled at this point! */
11165	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11166	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11167	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11168	/* Interrupts are re-enabled at this point! */
11169
11170	/*
11171	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
11172	*/
11173	if (RT_SUCCESS(rcRun))
11174	{ /* very likely */ }
11175	else
11176	{
11177	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11178	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11179	return rcRun;
11180	}
11181
11182	/*
11183	* Profile the VM-exit.
11184	*/
11185	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11186	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11187	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11188	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11189	HMVMX_START_EXIT_DISPATCH_PROF();
11190
11191	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11192
11193	/*
11194	* Handle the VM-exit.
11195	*/
11196	#ifdef HMVMX_USE_FUNCTION_TABLE
11197	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
11198	#else
11199	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
11200	#endif
11201	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11202	if (rcStrict == VINF_SUCCESS)
11203	{
11204	if (++(*pcLoops) <= cMaxResumeLoops)
11205	continue;
11206	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11207	rcStrict = VINF_EM_RAW_INTERRUPT;
11208	}
11209	break;
11210	}
11211
11212	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11213	return rcStrict;
11214	}
11215
11216
11217	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11218	/**
11219	* Runs the nested-guest code using hardware-assisted VMX.
11220	*
11221	* @returns VBox status code.
11222	* @param pVCpu The cross context virtual CPU structure.
11223	* @param pcLoops Pointer to the number of executed loops.
11224	*
11225	* @sa hmR0VmxRunGuestCodeNormal.
11226	*/
11227	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPU pVCpu, uint32_t *pcLoops)
11228	{
11229	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11230	Assert(pcLoops);
11231	Assert(*pcLoops <= cMaxResumeLoops);
11232	Assert(CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11233
11234	/*
11235	* Switch to the nested-guest VMCS as we may have transitioned from executing the
11236	* guest without leaving ring-0. Otherwise, if we came from ring-3 we would have
11237	* loaded the nested-guest VMCS while entering the VMX ring-0 session.
11238	*/
11239	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
11240	{
11241	int rc = hmR0VmxSwitchToGstOrNstGstVmcs(pVCpu, true /* fSwitchToNstGstVmcs */);
11242	if (RT_SUCCESS(rc))
11243	{ /* likely */ }
11244	else
11245	{
11246	LogRelFunc(("Failed to switch to the nested-guest VMCS. rc=%Rrc\n", rc));
11247	return rc;
11248	}
11249	}
11250
11251	VMXTRANSIENT VmxTransient;
11252	RT_ZERO(VmxTransient);
11253	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11254	VmxTransient.fIsNestedGuest = true;
11255
11256	/* Paranoia. */
11257	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfoNstGst);
11258
11259	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11260	for (;;)
11261	{
11262	Assert(!HMR0SuspendPending());
11263	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11264	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11265
11266	/*
11267	* Preparatory work for running guest code, this may force us to
11268	* return to ring-3.
11269	*
11270	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11271	*/
11272	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
11273	if (rcStrict != VINF_SUCCESS)
11274	break;
11275
11276	/* Interrupts are disabled at this point! */
11277	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11278	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11279	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11280	/* Interrupts are re-enabled at this point! */
11281
11282	/*
11283	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
11284	*/
11285	if (RT_SUCCESS(rcRun))
11286	{ /* very likely */ }
11287	else
11288	{
11289	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11290	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11291	return rcRun;
11292	}
11293
11294	/*
11295	* Profile the VM-exit.
11296	*/
11297	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11298	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11299	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitAll);
11300	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11301	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11302	HMVMX_START_EXIT_DISPATCH_PROF();
11303
11304	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11305
11306	/*
11307	* Handle the VM-exit.
11308	*/
11309	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
11310	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11311	if (rcStrict == VINF_SUCCESS)
11312	{
11313	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
11314	{
11315	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
11316	rcStrict = VINF_VMX_VMEXIT;
11317	}
11318	else
11319	{
11320	if (++(*pcLoops) <= cMaxResumeLoops)
11321	continue;
11322	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11323	rcStrict = VINF_EM_RAW_INTERRUPT;
11324	}
11325	}
11326	else
11327	Assert(rcStrict != VINF_VMX_VMEXIT);
11328	break;
11329	}
11330
11331	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11332	return rcStrict;
11333	}
11334	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
11335
11336
11337	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
11338	* probes.
11339	*
11340	* The following few functions and associated structure contains the bloat
11341	* necessary for providing detailed debug events and dtrace probes as well as
11342	* reliable host side single stepping. This works on the principle of
11343	* "subclassing" the normal execution loop and workers. We replace the loop
11344	* method completely and override selected helpers to add necessary adjustments
11345	* to their core operation.
11346	*
11347	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
11348	* any performance for debug and analysis features.
11349	*
11350	* @{
11351	*/
11352
11353	/**
11354	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
11355	* the debug run loop.
11356	*/
11357	typedef struct VMXRUNDBGSTATE
11358	{
11359	/** The RIP we started executing at. This is for detecting that we stepped. */
11360	uint64_t uRipStart;
11361	/** The CS we started executing with. */
11362	uint16_t uCsStart;
11363
11364	/** Whether we've actually modified the 1st execution control field. */
11365	bool fModifiedProcCtls : 1;
11366	/** Whether we've actually modified the 2nd execution control field. */
11367	bool fModifiedProcCtls2 : 1;
11368	/** Whether we've actually modified the exception bitmap. */
11369	bool fModifiedXcptBitmap : 1;
11370
11371	/** We desire the modified the CR0 mask to be cleared. */
11372	bool fClearCr0Mask : 1;
11373	/** We desire the modified the CR4 mask to be cleared. */
11374	bool fClearCr4Mask : 1;
11375	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
11376	uint32_t fCpe1Extra;
11377	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
11378	uint32_t fCpe1Unwanted;
11379	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
11380	uint32_t fCpe2Extra;
11381	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
11382	uint32_t bmXcptExtra;
11383	/** The sequence number of the Dtrace provider settings the state was
11384	* configured against. */
11385	uint32_t uDtraceSettingsSeqNo;
11386	/** VM-exits to check (one bit per VM-exit). */
11387	uint32_t bmExitsToCheck[3];
11388
11389	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
11390	uint32_t fProcCtlsInitial;
11391	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
11392	uint32_t fProcCtls2Initial;
11393	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
11394	uint32_t bmXcptInitial;
11395	} VMXRUNDBGSTATE;
11396	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
11397	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
11398
11399
11400	/**
11401	* Initializes the VMXRUNDBGSTATE structure.
11402	*
11403	* @param pVCpu The cross context virtual CPU structure of the
11404	* calling EMT.
11405	* @param pVmxTransient The VMX-transient structure.
11406	* @param pDbgState The debug state to initialize.
11407	*/
11408	static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11409	{
11410	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
11411	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
11412
11413	pDbgState->fModifiedProcCtls = false;
11414	pDbgState->fModifiedProcCtls2 = false;
11415	pDbgState->fModifiedXcptBitmap = false;
11416	pDbgState->fClearCr0Mask = false;
11417	pDbgState->fClearCr4Mask = false;
11418	pDbgState->fCpe1Extra = 0;
11419	pDbgState->fCpe1Unwanted = 0;
11420	pDbgState->fCpe2Extra = 0;
11421	pDbgState->bmXcptExtra = 0;
11422	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
11423	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
11424	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
11425	}
11426
11427
11428	/**
11429	* Updates the VMSC fields with changes requested by @a pDbgState.
11430	*
11431	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
11432	* immediately before executing guest code, i.e. when interrupts are disabled.
11433	* We don't check status codes here as we cannot easily assert or return in the
11434	* latter case.
11435	*
11436	* @param pVCpu The cross context virtual CPU structure.
11437	* @param pVmxTransient The VMX-transient structure.
11438	* @param pDbgState The debug state.
11439	*/
11440	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11441	{
11442	/*
11443	* Ensure desired flags in VMCS control fields are set.
11444	* (Ignoring write failure here, as we're committed and it's just debug extras.)
11445	*
11446	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
11447	* there should be no stale data in pCtx at this point.
11448	*/
11449	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11450	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11451	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11452	{
11453	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11454	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11455	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11456	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11457	pDbgState->fModifiedProcCtls = true;
11458	}
11459
11460	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11461	{
11462	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11463	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11464	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11465	pDbgState->fModifiedProcCtls2 = true;
11466	}
11467
11468	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11469	{
11470	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11471	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11472	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11473	pDbgState->fModifiedXcptBitmap = true;
11474	}
11475
11476	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11477	{
11478	pVmcsInfo->u64Cr0Mask = 0;
11479	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, 0);
11480	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11481	}
11482
11483	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11484	{
11485	pVmcsInfo->u64Cr4Mask = 0;
11486	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, 0);
11487	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11488	}
11489
11490	NOREF(pVCpu);
11491	}
11492
11493
11494	/**
11495	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11496	* re-entry next time around.
11497	*
11498	* @returns Strict VBox status code (i.e. informational status codes too).
11499	* @param pVCpu The cross context virtual CPU structure.
11500	* @param pVmxTransient The VMX-transient structure.
11501	* @param pDbgState The debug state.
11502	* @param rcStrict The return code from executing the guest using single
11503	* stepping.
11504	*/
11505	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11506	VBOXSTRICTRC rcStrict)
11507	{
11508	/*
11509	* Restore VM-exit control settings as we may not reenter this function the
11510	* next time around.
11511	*/
11512	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11513
11514	/* We reload the initial value, trigger what we can of recalculations the
11515	next time around. From the looks of things, that's all that's required atm. */
11516	if (pDbgState->fModifiedProcCtls)
11517	{
11518	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11519	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11520	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11521	AssertRCReturn(rc2, rc2);
11522	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11523	}
11524
11525	/* We're currently the only ones messing with this one, so just restore the
11526	cached value and reload the field. */
11527	if ( pDbgState->fModifiedProcCtls2
11528	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11529	{
11530	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
11531	AssertRCReturn(rc2, rc2);
11532	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
11533	}
11534
11535	/* If we've modified the exception bitmap, we restore it and trigger
11536	reloading and partial recalculation the next time around. */
11537	if (pDbgState->fModifiedXcptBitmap)
11538	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
11539
11540	return rcStrict;
11541	}
11542
11543
11544	/**
11545	* Configures VM-exit controls for current DBGF and DTrace settings.
11546	*
11547	* This updates @a pDbgState and the VMCS execution control fields to reflect
11548	* the necessary VM-exits demanded by DBGF and DTrace.
11549	*
11550	* @param pVCpu The cross context virtual CPU structure.
11551	* @param pVmxTransient The VMX-transient structure. May update
11552	* fUpdatedTscOffsettingAndPreemptTimer.
11553	* @param pDbgState The debug state.
11554	*/
11555	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11556	{
11557	/*
11558	* Take down the dtrace serial number so we can spot changes.
11559	*/
11560	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
11561	ASMCompilerBarrier();
11562
11563	/*
11564	* We'll rebuild most of the middle block of data members (holding the
11565	* current settings) as we go along here, so start by clearing it all.
11566	*/
11567	pDbgState->bmXcptExtra = 0;
11568	pDbgState->fCpe1Extra = 0;
11569	pDbgState->fCpe1Unwanted = 0;
11570	pDbgState->fCpe2Extra = 0;
11571	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
11572	pDbgState->bmExitsToCheck[i] = 0;
11573
11574	/*
11575	* Software interrupts (INT XXh) - no idea how to trigger these...
11576	*/
11577	PVM pVM = pVCpu->CTX_SUFF(pVM);
11578	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
11579	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
11580	{
11581	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11582	}
11583
11584	/*
11585	* INT3 breakpoints - triggered by #BP exceptions.
11586	*/
11587	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
11588	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11589
11590	/*
11591	* Exception bitmap and XCPT events+probes.
11592	*/
11593	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
11594	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
11595	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
11596
11597	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
11598	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
11599	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11600	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
11601	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
11602	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
11603	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
11604	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
11605	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
11606	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
11607	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
11608	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
11609	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
11610	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
11611	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
11612	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
11613	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
11614	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
11615
11616	if (pDbgState->bmXcptExtra)
11617	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11618
11619	/*
11620	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
11621	*
11622	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
11623	* So, when adding/changing/removing please don't forget to update it.
11624	*
11625	* Some of the macros are picking up local variables to save horizontal space,
11626	* (being able to see it in a table is the lesser evil here).
11627	*/
11628	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
11629	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
11630	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
11631	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
11632	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11633	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11634	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11635	} else do { } while (0)
11636	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
11637	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11638	{ \
11639	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
11640	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11641	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11642	} else do { } while (0)
11643	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
11644	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11645	{ \
11646	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
11647	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11648	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11649	} else do { } while (0)
11650	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
11651	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11652	{ \
11653	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
11654	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11655	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11656	} else do { } while (0)
11657
11658	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
11659	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
11660	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
11661	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
11662	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
11663
11664	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
11665	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
11666	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
11667	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
11668	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
11669	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
11670	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
11671	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
11672	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
11673	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
11674	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
11675	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
11676	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
11677	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
11678	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
11679	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
11680	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
11681	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
11682	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
11683	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
11684	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
11685	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
11686	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
11687	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
11688	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
11689	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
11690	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
11691	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
11692	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
11693	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
11694	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
11695	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
11696	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
11697	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
11698	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
11699	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
11700
11701	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
11702	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11703	{
11704	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
11705	\| CPUMCTX_EXTRN_APIC_TPR);
11706	AssertRC(rc);
11707
11708	#if 0 /** @todo fix me */
11709	pDbgState->fClearCr0Mask = true;
11710	pDbgState->fClearCr4Mask = true;
11711	#endif
11712	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
11713	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
11714	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11715	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
11716	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
11717	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
11718	require clearing here and in the loop if we start using it. */
11719	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
11720	}
11721	else
11722	{
11723	if (pDbgState->fClearCr0Mask)
11724	{
11725	pDbgState->fClearCr0Mask = false;
11726	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11727	}
11728	if (pDbgState->fClearCr4Mask)
11729	{
11730	pDbgState->fClearCr4Mask = false;
11731	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11732	}
11733	}
11734	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
11735	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
11736
11737	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
11738	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
11739	{
11740	/** @todo later, need to fix handler as it assumes this won't usually happen. */
11741	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
11742	}
11743	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
11744	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
11745
11746	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
11747	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
11748	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
11749	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
11750	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
11751	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
11752	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
11753	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
11754	#if 0 /** @todo too slow, fix handler. */
11755	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
11756	#endif
11757	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
11758
11759	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
11760	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
11761	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
11762	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
11763	{
11764	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11765	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
11766	}
11767	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11768	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11769	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11770	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11771
11772	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
11773	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
11774	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
11775	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
11776	{
11777	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11778	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
11779	}
11780	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
11781	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
11782	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
11783	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
11784
11785	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
11786	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
11787	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
11788	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
11789	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
11790	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
11791	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
11792	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
11793	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
11794	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
11795	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
11796	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
11797	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
11798	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
11799	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
11800	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
11801	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
11802	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
11803	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
11804	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
11805	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
11806	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
11807
11808	#undef IS_EITHER_ENABLED
11809	#undef SET_ONLY_XBM_IF_EITHER_EN
11810	#undef SET_CPE1_XBM_IF_EITHER_EN
11811	#undef SET_CPEU_XBM_IF_EITHER_EN
11812	#undef SET_CPE2_XBM_IF_EITHER_EN
11813
11814	/*
11815	* Sanitize the control stuff.
11816	*/
11817	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
11818	if (pDbgState->fCpe2Extra)
11819	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
11820	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
11821	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
11822	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
11823	{
11824	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
11825	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11826	}
11827
11828	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
11829	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
11830	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
11831	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
11832	}
11833
11834
11835	/**
11836	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
11837	* appropriate.
11838	*
11839	* The caller has checked the VM-exit against the
11840	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
11841	* already, so we don't have to do that either.
11842	*
11843	* @returns Strict VBox status code (i.e. informational status codes too).
11844	* @param pVCpu The cross context virtual CPU structure.
11845	* @param pVmxTransient The VMX-transient structure.
11846	* @param uExitReason The VM-exit reason.
11847	*
11848	* @remarks The name of this function is displayed by dtrace, so keep it short
11849	* and to the point. No longer than 33 chars long, please.
11850	*/
11851	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
11852	{
11853	/*
11854	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
11855	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
11856	*
11857	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
11858	* does. Must add/change/remove both places. Same ordering, please.
11859	*
11860	* Added/removed events must also be reflected in the next section
11861	* where we dispatch dtrace events.
11862	*/
11863	bool fDtrace1 = false;
11864	bool fDtrace2 = false;
11865	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
11866	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
11867	uint32_t uEventArg = 0;
11868	#define SET_EXIT(a_EventSubName) \
11869	do { \
11870	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11871	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11872	} while (0)
11873	#define SET_BOTH(a_EventSubName) \
11874	do { \
11875	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
11876	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11877	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
11878	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11879	} while (0)
11880	switch (uExitReason)
11881	{
11882	case VMX_EXIT_MTF:
11883	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11884
11885	case VMX_EXIT_XCPT_OR_NMI:
11886	{
11887	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11888	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
11889	{
11890	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11891	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
11892	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
11893	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
11894	{
11895	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
11896	{
11897	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11898	uEventArg = pVmxTransient->uExitIntErrorCode;
11899	}
11900	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
11901	switch (enmEvent1)
11902	{
11903	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
11904	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
11905	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
11906	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
11907	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
11908	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
11909	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
11910	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
11911	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
11912	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
11913	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
11914	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
11915	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
11916	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
11917	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
11918	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
11919	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
11920	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
11921	default: break;
11922	}
11923	}
11924	else
11925	AssertFailed();
11926	break;
11927
11928	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
11929	uEventArg = idxVector;
11930	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
11931	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
11932	break;
11933	}
11934	break;
11935	}
11936
11937	case VMX_EXIT_TRIPLE_FAULT:
11938	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
11939	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
11940	break;
11941	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
11942	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
11943	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
11944	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
11945	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
11946
11947	/* Instruction specific VM-exits: */
11948	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
11949	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
11950	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
11951	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
11952	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
11953	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
11954	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
11955	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
11956	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
11957	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
11958	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
11959	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
11960	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
11961	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
11962	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
11963	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
11964	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
11965	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
11966	case VMX_EXIT_MOV_CRX:
11967	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11968	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
11969	SET_BOTH(CRX_READ);
11970	else
11971	SET_BOTH(CRX_WRITE);
11972	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
11973	break;
11974	case VMX_EXIT_MOV_DRX:
11975	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11976	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
11977	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
11978	SET_BOTH(DRX_READ);
11979	else
11980	SET_BOTH(DRX_WRITE);
11981	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
11982	break;
11983	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
11984	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
11985	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
11986	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
11987	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
11988	case VMX_EXIT_GDTR_IDTR_ACCESS:
11989	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11990	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
11991	{
11992	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
11993	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
11994	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
11995	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
11996	}
11997	break;
11998
11999	case VMX_EXIT_LDTR_TR_ACCESS:
12000	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12001	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
12002	{
12003	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
12004	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
12005	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
12006	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
12007	}
12008	break;
12009
12010	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
12011	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
12012	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
12013	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
12014	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
12015	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
12016	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
12017	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
12018	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
12019	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
12020	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
12021
12022	/* Events that aren't relevant at this point. */
12023	case VMX_EXIT_EXT_INT:
12024	case VMX_EXIT_INT_WINDOW:
12025	case VMX_EXIT_NMI_WINDOW:
12026	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12027	case VMX_EXIT_PREEMPT_TIMER:
12028	case VMX_EXIT_IO_INSTR:
12029	break;
12030
12031	/* Errors and unexpected events. */
12032	case VMX_EXIT_INIT_SIGNAL:
12033	case VMX_EXIT_SIPI:
12034	case VMX_EXIT_IO_SMI:
12035	case VMX_EXIT_SMI:
12036	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12037	case VMX_EXIT_ERR_MSR_LOAD:
12038	case VMX_EXIT_ERR_MACHINE_CHECK:
12039	case VMX_EXIT_PML_FULL:
12040	case VMX_EXIT_VIRTUALIZED_EOI:
12041	break;
12042
12043	default:
12044	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12045	break;
12046	}
12047	#undef SET_BOTH
12048	#undef SET_EXIT
12049
12050	/*
12051	* Dtrace tracepoints go first. We do them here at once so we don't
12052	* have to copy the guest state saving and stuff a few dozen times.
12053	* Down side is that we've got to repeat the switch, though this time
12054	* we use enmEvent since the probes are a subset of what DBGF does.
12055	*/
12056	if (fDtrace1 \|\| fDtrace2)
12057	{
12058	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12059	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12060	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12061	switch (enmEvent1)
12062	{
12063	/** @todo consider which extra parameters would be helpful for each probe. */
12064	case DBGFEVENT_END: break;
12065	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
12066	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
12067	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
12068	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
12069	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
12070	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
12071	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
12072	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
12073	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
12074	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
12075	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
12076	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
12077	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
12078	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
12079	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
12080	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
12081	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
12082	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
12083	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12084	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
12085	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
12086	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
12087	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
12088	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
12089	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
12090	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
12091	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
12092	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12093	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12094	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12095	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12096	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
12097	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
12098	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
12099	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
12100	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
12101	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
12102	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
12103	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
12104	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
12105	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
12106	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
12107	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
12108	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
12109	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
12110	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
12111	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
12112	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
12113	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
12114	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
12115	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
12116	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
12117	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
12118	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
12119	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
12120	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
12121	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
12122	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
12123	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
12124	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
12125	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
12126	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
12127	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
12128	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
12129	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
12130	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
12131	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
12132	}
12133	switch (enmEvent2)
12134	{
12135	/** @todo consider which extra parameters would be helpful for each probe. */
12136	case DBGFEVENT_END: break;
12137	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
12138	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
12139	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
12140	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
12141	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
12142	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
12143	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
12144	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
12145	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
12146	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12147	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12148	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12149	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12150	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
12151	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
12152	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
12153	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
12154	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
12155	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
12156	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
12157	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
12158	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
12159	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
12160	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
12161	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
12162	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
12163	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
12164	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
12165	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
12166	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
12167	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
12168	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
12169	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
12170	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
12171	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
12172	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
12173	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
12174	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
12175	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
12176	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
12177	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
12178	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
12179	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
12180	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
12181	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
12182	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
12183	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
12184	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
12185	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
12186	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
12187	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
12188	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
12189	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
12190	}
12191	}
12192
12193	/*
12194	* Fire of the DBGF event, if enabled (our check here is just a quick one,
12195	* the DBGF call will do a full check).
12196	*
12197	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
12198	* Note! If we have to events, we prioritize the first, i.e. the instruction
12199	* one, in order to avoid event nesting.
12200	*/
12201	PVM pVM = pVCpu->CTX_SUFF(pVM);
12202	if ( enmEvent1 != DBGFEVENT_END
12203	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
12204	{
12205	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12206	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
12207	if (rcStrict != VINF_SUCCESS)
12208	return rcStrict;
12209	}
12210	else if ( enmEvent2 != DBGFEVENT_END
12211	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
12212	{
12213	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12214	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
12215	if (rcStrict != VINF_SUCCESS)
12216	return rcStrict;
12217	}
12218
12219	return VINF_SUCCESS;
12220	}
12221
12222
12223	/**
12224	* Single-stepping VM-exit filtering.
12225	*
12226	* This is preprocessing the VM-exits and deciding whether we've gotten far
12227	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
12228	* handling is performed.
12229	*
12230	* @returns Strict VBox status code (i.e. informational status codes too).
12231	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
12232	* @param pVmxTransient The VMX-transient structure.
12233	* @param pDbgState The debug state.
12234	*/
12235	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
12236	{
12237	/*
12238	* Expensive (saves context) generic dtrace VM-exit probe.
12239	*/
12240	uint32_t const uExitReason = pVmxTransient->uExitReason;
12241	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
12242	{ /* more likely */ }
12243	else
12244	{
12245	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12246	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12247	AssertRC(rc);
12248	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
12249	}
12250
12251	/*
12252	* Check for host NMI, just to get that out of the way.
12253	*/
12254	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
12255	{ /* normally likely */ }
12256	else
12257	{
12258	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12259	AssertRCReturn(rc2, rc2);
12260	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12261	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
12262	return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
12263	}
12264
12265	/*
12266	* Check for single stepping event if we're stepping.
12267	*/
12268	if (pVCpu->hm.s.fSingleInstruction)
12269	{
12270	switch (uExitReason)
12271	{
12272	case VMX_EXIT_MTF:
12273	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
12274
12275	/* Various events: */
12276	case VMX_EXIT_XCPT_OR_NMI:
12277	case VMX_EXIT_EXT_INT:
12278	case VMX_EXIT_TRIPLE_FAULT:
12279	case VMX_EXIT_INT_WINDOW:
12280	case VMX_EXIT_NMI_WINDOW:
12281	case VMX_EXIT_TASK_SWITCH:
12282	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12283	case VMX_EXIT_APIC_ACCESS:
12284	case VMX_EXIT_EPT_VIOLATION:
12285	case VMX_EXIT_EPT_MISCONFIG:
12286	case VMX_EXIT_PREEMPT_TIMER:
12287
12288	/* Instruction specific VM-exits: */
12289	case VMX_EXIT_CPUID:
12290	case VMX_EXIT_GETSEC:
12291	case VMX_EXIT_HLT:
12292	case VMX_EXIT_INVD:
12293	case VMX_EXIT_INVLPG:
12294	case VMX_EXIT_RDPMC:
12295	case VMX_EXIT_RDTSC:
12296	case VMX_EXIT_RSM:
12297	case VMX_EXIT_VMCALL:
12298	case VMX_EXIT_VMCLEAR:
12299	case VMX_EXIT_VMLAUNCH:
12300	case VMX_EXIT_VMPTRLD:
12301	case VMX_EXIT_VMPTRST:
12302	case VMX_EXIT_VMREAD:
12303	case VMX_EXIT_VMRESUME:
12304	case VMX_EXIT_VMWRITE:
12305	case VMX_EXIT_VMXOFF:
12306	case VMX_EXIT_VMXON:
12307	case VMX_EXIT_MOV_CRX:
12308	case VMX_EXIT_MOV_DRX:
12309	case VMX_EXIT_IO_INSTR:
12310	case VMX_EXIT_RDMSR:
12311	case VMX_EXIT_WRMSR:
12312	case VMX_EXIT_MWAIT:
12313	case VMX_EXIT_MONITOR:
12314	case VMX_EXIT_PAUSE:
12315	case VMX_EXIT_GDTR_IDTR_ACCESS:
12316	case VMX_EXIT_LDTR_TR_ACCESS:
12317	case VMX_EXIT_INVEPT:
12318	case VMX_EXIT_RDTSCP:
12319	case VMX_EXIT_INVVPID:
12320	case VMX_EXIT_WBINVD:
12321	case VMX_EXIT_XSETBV:
12322	case VMX_EXIT_RDRAND:
12323	case VMX_EXIT_INVPCID:
12324	case VMX_EXIT_VMFUNC:
12325	case VMX_EXIT_RDSEED:
12326	case VMX_EXIT_XSAVES:
12327	case VMX_EXIT_XRSTORS:
12328	{
12329	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12330	AssertRCReturn(rc, rc);
12331	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
12332	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
12333	return VINF_EM_DBG_STEPPED;
12334	break;
12335	}
12336
12337	/* Errors and unexpected events: */
12338	case VMX_EXIT_INIT_SIGNAL:
12339	case VMX_EXIT_SIPI:
12340	case VMX_EXIT_IO_SMI:
12341	case VMX_EXIT_SMI:
12342	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12343	case VMX_EXIT_ERR_MSR_LOAD:
12344	case VMX_EXIT_ERR_MACHINE_CHECK:
12345	case VMX_EXIT_PML_FULL:
12346	case VMX_EXIT_VIRTUALIZED_EOI:
12347	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
12348	break;
12349
12350	default:
12351	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12352	break;
12353	}
12354	}
12355
12356	/*
12357	* Check for debugger event breakpoints and dtrace probes.
12358	*/
12359	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
12360	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
12361	{
12362	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
12363	if (rcStrict != VINF_SUCCESS)
12364	return rcStrict;
12365	}
12366
12367	/*
12368	* Normal processing.
12369	*/
12370	#ifdef HMVMX_USE_FUNCTION_TABLE
12371	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
12372	#else
12373	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
12374	#endif
12375	}
12376
12377
12378	/**
12379	* Single steps guest code using hardware-assisted VMX.
12380	*
12381	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
12382	* but single-stepping through the hypervisor debugger.
12383	*
12384	* @returns Strict VBox status code (i.e. informational status codes too).
12385	* @param pVCpu The cross context virtual CPU structure.
12386	* @param pcLoops Pointer to the number of executed loops.
12387	*
12388	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
12389	*/
12390	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, uint32_t *pcLoops)
12391	{
12392	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
12393	Assert(pcLoops);
12394	Assert(*pcLoops <= cMaxResumeLoops);
12395
12396	VMXTRANSIENT VmxTransient;
12397	RT_ZERO(VmxTransient);
12398	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
12399
12400	/* Set HMCPU indicators. */
12401	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
12402	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
12403	pVCpu->hm.s.fDebugWantRdTscExit = false;
12404	pVCpu->hm.s.fUsingDebugLoop = true;
12405
12406	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
12407	VMXRUNDBGSTATE DbgState;
12408	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
12409	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12410
12411	/*
12412	* The loop.
12413	*/
12414	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
12415	for (;;)
12416	{
12417	Assert(!HMR0SuspendPending());
12418	HMVMX_ASSERT_CPU_SAFE(pVCpu);
12419	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
12420	bool fStepping = pVCpu->hm.s.fSingleInstruction;
12421
12422	/* Set up VM-execution controls the next two can respond to. */
12423	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12424
12425	/*
12426	* Preparatory work for running guest code, this may force us to
12427	* return to ring-3.
12428	*
12429	* Warning! This bugger disables interrupts on VINF_SUCCESS!
12430	*/
12431	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
12432	if (rcStrict != VINF_SUCCESS)
12433	break;
12434
12435	/* Interrupts are disabled at this point! */
12436	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
12437
12438	/* Override any obnoxious code in the above two calls. */
12439	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12440
12441	/*
12442	* Finally execute the guest.
12443	*/
12444	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
12445
12446	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
12447	/* Interrupts are re-enabled at this point! */
12448
12449	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
12450	if (RT_SUCCESS(rcRun))
12451	{ /* very likely */ }
12452	else
12453	{
12454	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12455	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12456	return rcRun;
12457	}
12458
12459	/* Profile the VM-exit. */
12460	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12461	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12462	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12463	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12464	HMVMX_START_EXIT_DISPATCH_PROF();
12465
12466	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12467
12468	/*
12469	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12470	*/
12471	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12472	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12473	if (rcStrict != VINF_SUCCESS)
12474	break;
12475	if (++(*pcLoops) > cMaxResumeLoops)
12476	{
12477	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12478	rcStrict = VINF_EM_RAW_INTERRUPT;
12479	break;
12480	}
12481
12482	/*
12483	* Stepping: Did the RIP change, if so, consider it a single step.
12484	* Otherwise, make sure one of the TFs gets set.
12485	*/
12486	if (fStepping)
12487	{
12488	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12489	AssertRC(rc);
12490	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12491	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12492	{
12493	rcStrict = VINF_EM_DBG_STEPPED;
12494	break;
12495	}
12496	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12497	}
12498
12499	/*
12500	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12501	*/
12502	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12503	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12504	}
12505
12506	/*
12507	* Clear the X86_EFL_TF if necessary.
12508	*/
12509	if (pVCpu->hm.s.fClearTrapFlag)
12510	{
12511	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12512	AssertRC(rc);
12513	pVCpu->hm.s.fClearTrapFlag = false;
12514	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12515	}
12516	/** @todo there seems to be issues with the resume flag when the monitor trap
12517	* flag is pending without being used. Seen early in bios init when
12518	* accessing APIC page in protected mode. */
12519
12520	/*
12521	* Restore VM-exit control settings as we may not re-enter this function the
12522	* next time around.
12523	*/
12524	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12525
12526	/* Restore HMCPU indicators. */
12527	pVCpu->hm.s.fUsingDebugLoop = false;
12528	pVCpu->hm.s.fDebugWantRdTscExit = false;
12529	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
12530
12531	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
12532	return rcStrict;
12533	}
12534
12535
12536	/** @} */
12537
12538
12539	/**
12540	* Checks if any expensive dtrace probes are enabled and we should go to the
12541	* debug loop.
12542	*
12543	* @returns true if we should use debug loop, false if not.
12544	*/
12545	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
12546	{
12547	/* It's probably faster to OR the raw 32-bit counter variables together.
12548	Since the variables are in an array and the probes are next to one
12549	another (more or less), we have good locality. So, better read
12550	eight-nine cache lines ever time and only have one conditional, than
12551	128+ conditionals, right? */
12552	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
12553	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
12554	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
12555	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
12556	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
12557	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
12558	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
12559	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
12560	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
12561	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
12562	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
12563	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
12564	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
12565	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
12566	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
12567	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
12568	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
12569	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
12570	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
12571	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
12572	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
12573	) != 0
12574	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
12575	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
12576	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
12577	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
12578	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
12579	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
12580	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
12581	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
12582	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
12583	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
12584	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
12585	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
12586	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
12587	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
12588	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
12589	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
12590	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
12591	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
12592	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
12593	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
12594	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
12595	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
12596	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
12597	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
12598	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
12599	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
12600	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
12601	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
12602	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
12603	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
12604	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
12605	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
12606	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
12607	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
12608	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
12609	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
12610	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
12611	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
12612	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
12613	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
12614	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
12615	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
12616	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
12617	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
12618	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
12619	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
12620	) != 0
12621	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
12622	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
12623	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
12624	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
12625	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
12626	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
12627	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
12628	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
12629	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
12630	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
12631	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
12632	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
12633	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
12634	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
12635	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
12636	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
12637	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
12638	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
12639	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
12640	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
12641	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
12642	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
12643	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
12644	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
12645	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
12646	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
12647	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
12648	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
12649	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
12650	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
12651	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
12652	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
12653	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
12654	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
12655	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
12656	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
12657	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
12658	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
12659	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
12660	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
12661	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
12662	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
12663	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
12664	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
12665	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
12666	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
12667	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
12668	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
12669	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
12670	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
12671	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
12672	) != 0;
12673	}
12674
12675
12676	/**
12677	* Runs the guest using hardware-assisted VMX.
12678	*
12679	* @returns Strict VBox status code (i.e. informational status codes too).
12680	* @param pVCpu The cross context virtual CPU structure.
12681	*/
12682	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
12683	{
12684	AssertPtr(pVCpu);
12685	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12686	Assert(VMMRZCallRing3IsEnabled(pVCpu));
12687	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12688	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
12689
12690	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
12691
12692	VBOXSTRICTRC rcStrict;
12693	uint32_t cLoops = 0;
12694	for (;;)
12695	{
12696	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12697	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
12698	#else
12699	bool const fInNestedGuestMode = false;
12700	#endif
12701	if (!fInNestedGuestMode)
12702	{
12703	if ( !pVCpu->hm.s.fUseDebugLoop
12704	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
12705	&& !DBGFIsStepping(pVCpu)
12706	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
12707	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
12708	else
12709	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
12710	}
12711	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12712	else
12713	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
12714
12715	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
12716	{
12717	Assert(CPUMIsGuestInVmxNonRootMode(pCtx));
12718	continue;
12719	}
12720	if (rcStrict == VINF_VMX_VMEXIT)
12721	{
12722	Assert(!CPUMIsGuestInVmxNonRootMode(pCtx));
12723	continue;
12724	}
12725	#endif
12726	break;
12727	}
12728
12729	int const rcLoop = VBOXSTRICTRC_VAL(rcStrict);
12730	switch (rcLoop)
12731	{
12732	case VERR_EM_INTERPRETER: rcStrict = VINF_EM_RAW_EMULATE_INSTR; break;
12733	case VINF_EM_RESET: rcStrict = VINF_EM_TRIPLE_FAULT; break;
12734	}
12735
12736	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
12737	if (RT_FAILURE(rc2))
12738	{
12739	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
12740	rcStrict = rc2;
12741	}
12742	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12743	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
12744	return rcStrict;
12745	}
12746
12747
12748	#ifndef HMVMX_USE_FUNCTION_TABLE
12749	/**
12750	* Handles a guest VM-exit from hardware-assisted VMX execution.
12751	*
12752	* @returns Strict VBox status code (i.e. informational status codes too).
12753	* @param pVCpu The cross context virtual CPU structure.
12754	* @param pVmxTransient The VMX-transient structure.
12755	*/
12756	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12757	{
12758	#ifdef DEBUG_ramshankar
12759	#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
12760	do { \
12761	if (a_fSave != 0) \
12762	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL); \
12763	VBOXSTRICTRC rcStrict = a_CallExpr; \
12764	if (a_fSave != 0) \
12765	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
12766	return rcStrict; \
12767	} while (0)
12768	#else
12769	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
12770	#endif
12771	uint32_t const uExitReason = pVmxTransient->uExitReason;
12772	switch (uExitReason)
12773	{
12774	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
12775	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
12776	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
12777	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
12778	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
12779	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
12780	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
12781	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
12782	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
12783	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
12784	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
12785	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
12786	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
12787	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
12788	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
12789	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
12790	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
12791	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
12792	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
12793	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
12794	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
12795	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
12796	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
12797	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
12798	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
12799	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
12800	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
12801	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
12802	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
12803	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
12804	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12805	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
12806	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
12807	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
12808	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
12809	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
12810	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
12811	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
12812	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
12813	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
12814	case VMX_EXIT_INVVPID: VMEXIT_CALL_RET(0, hmR0VmxExitInvvpid(pVCpu, pVmxTransient));
12815	case VMX_EXIT_INVEPT: VMEXIT_CALL_RET(0, hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient));
12816	#else
12817	case VMX_EXIT_VMCLEAR:
12818	case VMX_EXIT_VMLAUNCH:
12819	case VMX_EXIT_VMPTRLD:
12820	case VMX_EXIT_VMPTRST:
12821	case VMX_EXIT_VMREAD:
12822	case VMX_EXIT_VMRESUME:
12823	case VMX_EXIT_VMWRITE:
12824	case VMX_EXIT_VMXOFF:
12825	case VMX_EXIT_VMXON:
12826	case VMX_EXIT_INVVPID:
12827	case VMX_EXIT_INVEPT:
12828	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12829	#endif
12830
12831	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
12832	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
12833	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
12834
12835	case VMX_EXIT_INIT_SIGNAL:
12836	case VMX_EXIT_SIPI:
12837	case VMX_EXIT_IO_SMI:
12838	case VMX_EXIT_SMI:
12839	case VMX_EXIT_ERR_MSR_LOAD:
12840	case VMX_EXIT_ERR_MACHINE_CHECK:
12841	case VMX_EXIT_PML_FULL:
12842	case VMX_EXIT_VIRTUALIZED_EOI:
12843	case VMX_EXIT_GDTR_IDTR_ACCESS:
12844	case VMX_EXIT_LDTR_TR_ACCESS:
12845	case VMX_EXIT_APIC_WRITE:
12846	case VMX_EXIT_RDRAND:
12847	case VMX_EXIT_RSM:
12848	case VMX_EXIT_VMFUNC:
12849	case VMX_EXIT_ENCLS:
12850	case VMX_EXIT_RDSEED:
12851	case VMX_EXIT_XSAVES:
12852	case VMX_EXIT_XRSTORS:
12853	case VMX_EXIT_UMWAIT:
12854	case VMX_EXIT_TPAUSE:
12855	default:
12856	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12857	}
12858	#undef VMEXIT_CALL_RET
12859	}
12860	#endif /* !HMVMX_USE_FUNCTION_TABLE */
12861
12862
12863	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12864	/**
12865	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
12866	*
12867	* @returns Strict VBox status code (i.e. informational status codes too).
12868	* @param pVCpu The cross context virtual CPU structure.
12869	* @param pVmxTransient The VMX-transient structure.
12870	*/
12871	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12872	{
12873	uint32_t const uExitReason = pVmxTransient->uExitReason;
12874	switch (uExitReason)
12875	{
12876	case VMX_EXIT_EPT_MISCONFIG: return hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient);
12877	case VMX_EXIT_EPT_VIOLATION: return hmR0VmxExitEptViolation(pVCpu, pVmxTransient);
12878	case VMX_EXIT_XCPT_OR_NMI: return hmR0VmxExitXcptOrNmiNested(pVCpu, pVmxTransient);
12879	case VMX_EXIT_IO_INSTR: return hmR0VmxExitIoInstrNested(pVCpu, pVmxTransient);
12880	case VMX_EXIT_HLT: return hmR0VmxExitHltNested(pVCpu, pVmxTransient);
12881
12882	/*
12883	* We shouldn't direct host physical interrupts to the nested-guest.
12884	*/
12885	case VMX_EXIT_EXT_INT:
12886	return hmR0VmxExitExtInt(pVCpu, pVmxTransient);
12887
12888	/*
12889	* Instructions that cause VM-exits unconditionally or the condition is
12890	* always is taken solely from the guest hypervisor (meaning if the VM-exit
12891	* happens, it's guaranteed to be a nested-guest VM-exit).
12892	*
12893	* - Provides VM-exit instruction length ONLY.
12894	*/
12895	case VMX_EXIT_CPUID: /* Unconditional. */
12896	case VMX_EXIT_VMCALL:
12897	case VMX_EXIT_GETSEC:
12898	case VMX_EXIT_INVD:
12899	case VMX_EXIT_XSETBV:
12900	case VMX_EXIT_VMLAUNCH:
12901	case VMX_EXIT_VMRESUME:
12902	case VMX_EXIT_VMXOFF:
12903	case VMX_EXIT_ENCLS: /* Condition specified solely by guest hypervisor. */
12904	case VMX_EXIT_VMFUNC:
12905	return hmR0VmxExitInstrNested(pVCpu, pVmxTransient);
12906
12907	/*
12908	* Instructions that cause VM-exits unconditionally or the condition is
12909	* always is taken solely from the guest hypervisor (meaning if the VM-exit
12910	* happens, it's guaranteed to be a nested-guest VM-exit).
12911	*
12912	* - Provides VM-exit instruction length.
12913	* - Provides VM-exit information.
12914	* - Optionally provides Exit qualification.
12915	*
12916	* Since Exit qualification is 0 for all VM-exits where it is not
12917	* applicable, reading and passing it to the guest should produce
12918	* defined behavior.
12919	*
12920	* See Intel spec. 27.2.1 "Basic VM-Exit Information".
12921	*/
12922	case VMX_EXIT_INVEPT: /* Unconditional. */
12923	case VMX_EXIT_INVVPID:
12924	case VMX_EXIT_VMCLEAR:
12925	case VMX_EXIT_VMPTRLD:
12926	case VMX_EXIT_VMPTRST:
12927	case VMX_EXIT_VMXON:
12928	case VMX_EXIT_GDTR_IDTR_ACCESS: /* Condition specified solely by guest hypervisor. */
12929	case VMX_EXIT_LDTR_TR_ACCESS:
12930	case VMX_EXIT_RDRAND:
12931	case VMX_EXIT_RDSEED:
12932	case VMX_EXIT_XSAVES:
12933	case VMX_EXIT_XRSTORS:
12934	case VMX_EXIT_UMWAIT:
12935	case VMX_EXIT_TPAUSE:
12936	return hmR0VmxExitInstrWithInfoNested(pVCpu, pVmxTransient);
12937
12938	case VMX_EXIT_RDTSC: return hmR0VmxExitRdtscNested(pVCpu, pVmxTransient);
12939	case VMX_EXIT_RDTSCP: return hmR0VmxExitRdtscpNested(pVCpu, pVmxTransient);
12940	case VMX_EXIT_RDMSR: return hmR0VmxExitRdmsrNested(pVCpu, pVmxTransient);
12941	case VMX_EXIT_WRMSR: return hmR0VmxExitWrmsrNested(pVCpu, pVmxTransient);
12942	case VMX_EXIT_INVLPG: return hmR0VmxExitInvlpgNested(pVCpu, pVmxTransient);
12943	case VMX_EXIT_INVPCID: return hmR0VmxExitInvpcidNested(pVCpu, pVmxTransient);
12944	case VMX_EXIT_TASK_SWITCH: return hmR0VmxExitTaskSwitchNested(pVCpu, pVmxTransient);
12945	case VMX_EXIT_WBINVD: return hmR0VmxExitWbinvdNested(pVCpu, pVmxTransient);
12946	case VMX_EXIT_MTF: return hmR0VmxExitMtfNested(pVCpu, pVmxTransient);
12947	case VMX_EXIT_APIC_ACCESS: return hmR0VmxExitApicAccessNested(pVCpu, pVmxTransient);
12948	case VMX_EXIT_APIC_WRITE: return hmR0VmxExitApicWriteNested(pVCpu, pVmxTransient);
12949	case VMX_EXIT_VIRTUALIZED_EOI: return hmR0VmxExitVirtEoiNested(pVCpu, pVmxTransient);
12950	case VMX_EXIT_MOV_CRX: return hmR0VmxExitMovCRxNested(pVCpu, pVmxTransient);
12951	case VMX_EXIT_INT_WINDOW: return hmR0VmxExitIntWindowNested(pVCpu, pVmxTransient);
12952	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindowNested(pVCpu, pVmxTransient);
12953	case VMX_EXIT_TPR_BELOW_THRESHOLD: return hmR0VmxExitTprBelowThresholdNested(pVCpu, pVmxTransient);
12954	case VMX_EXIT_MWAIT: return hmR0VmxExitMwaitNested(pVCpu, pVmxTransient);
12955	case VMX_EXIT_MONITOR: return hmR0VmxExitMonitorNested(pVCpu, pVmxTransient);
12956	case VMX_EXIT_PAUSE: return hmR0VmxExitPauseNested(pVCpu, pVmxTransient);
12957
12958	case VMX_EXIT_PREEMPT_TIMER:
12959	{
12960	/** @todo NSTVMX: Preempt timer. */
12961	return hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient);
12962	}
12963
12964	case VMX_EXIT_MOV_DRX: return hmR0VmxExitMovDRxNested(pVCpu, pVmxTransient);
12965	case VMX_EXIT_RDPMC: return hmR0VmxExitRdpmcNested(pVCpu, pVmxTransient);
12966
12967	case VMX_EXIT_VMREAD:
12968	case VMX_EXIT_VMWRITE: return hmR0VmxExitVmreadVmwriteNested(pVCpu, pVmxTransient);
12969
12970	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFaultNested(pVCpu, pVmxTransient);
12971	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestStateNested(pVCpu, pVmxTransient);
12972
12973	case VMX_EXIT_INIT_SIGNAL:
12974	case VMX_EXIT_SIPI:
12975	case VMX_EXIT_IO_SMI:
12976	case VMX_EXIT_SMI:
12977	case VMX_EXIT_ERR_MSR_LOAD:
12978	case VMX_EXIT_ERR_MACHINE_CHECK:
12979	case VMX_EXIT_PML_FULL:
12980	case VMX_EXIT_RSM:
12981	default:
12982	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12983	}
12984	}
12985	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
12986
12987
12988	/** @name VM-exit helpers.
12989	* @{
12990	*/
12991	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12992	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit helpers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
12993	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12994
12995	/** Macro for VM-exits called unexpectedly. */
12996	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_HmError) \
12997	do { \
12998	(a_pVCpu)->hm.s.u32HMError = (a_HmError); \
12999	return VERR_VMX_UNEXPECTED_EXIT; \
13000	} while (0)
13001
13002	#ifdef VBOX_STRICT
13003	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
13004	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
13005	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
13006
13007	# define HMVMX_ASSERT_PREEMPT_CPUID() \
13008	do { \
13009	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
13010	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
13011	} while (0)
13012
13013	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13014	do { \
13015	AssertPtr((a_pVCpu)); \
13016	AssertPtr((a_pVmxTransient)); \
13017	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
13018	Assert((a_pVmxTransient)->pVmcsInfo); \
13019	Assert(ASMIntAreEnabled()); \
13020	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
13021	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
13022	Log4Func(("vcpu[%RU32]\n", (a_pVCpu)->idCpu)); \
13023	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
13024	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
13025	HMVMX_ASSERT_PREEMPT_CPUID(); \
13026	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
13027	} while (0)
13028
13029	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13030	do { \
13031	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); \
13032	Assert((a_pVmxTransient)->fIsNestedGuest); \
13033	} while (0)
13034
13035	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13036	do { \
13037	Log4Func(("\n")); \
13038	} while (0)
13039	#else
13040	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13041	do { \
13042	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
13043	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
13044	} while (0)
13045
13046	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13047	do { HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); } while (0)
13048
13049	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
13050	#endif
13051
13052	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13053	/** Macro that does the necessary privilege checks and intercepted VM-exits for
13054	* guests that attempted to execute a VMX instruction. */
13055	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
13056	do \
13057	{ \
13058	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
13059	if (rcStrictTmp == VINF_SUCCESS) \
13060	{ /* likely */ } \
13061	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
13062	{ \
13063	Assert((a_pVCpu)->hm.s.Event.fPending); \
13064	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
13065	return VINF_SUCCESS; \
13066	} \
13067	else \
13068	{ \
13069	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
13070	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
13071	} \
13072	} while (0)
13073
13074	/** Macro that decodes a memory operand for an VM-exit caused by an instruction. */
13075	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
13076	do \
13077	{ \
13078	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
13079	(a_pGCPtrEffAddr)); \
13080	if (rcStrictTmp == VINF_SUCCESS) \
13081	{ /* likely */ } \
13082	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
13083	{ \
13084	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
13085	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
13086	NOREF(uXcptTmp); \
13087	return VINF_SUCCESS; \
13088	} \
13089	else \
13090	{ \
13091	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
13092	return rcStrictTmp; \
13093	} \
13094	} while (0)
13095	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13096
13097
13098	/**
13099	* Advances the guest RIP by the specified number of bytes.
13100	*
13101	* @param pVCpu The cross context virtual CPU structure.
13102	* @param cbInstr Number of bytes to advance the RIP by.
13103	*
13104	* @remarks No-long-jump zone!!!
13105	*/
13106	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
13107	{
13108	/* Advance the RIP. */
13109	pVCpu->cpum.GstCtx.rip += cbInstr;
13110	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
13111
13112	/* Update interrupt inhibition. */
13113	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
13114	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
13115	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
13116	}
13117
13118
13119	/**
13120	* Advances the guest RIP after reading it from the VMCS.
13121	*
13122	* @returns VBox status code, no informational status codes.
13123	* @param pVCpu The cross context virtual CPU structure.
13124	* @param pVmxTransient The VMX-transient structure.
13125	*
13126	* @remarks No-long-jump zone!!!
13127	*/
13128	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13129	{
13130	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13131	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
13132	AssertRCReturn(rc, rc);
13133
13134	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
13135	return VINF_SUCCESS;
13136	}
13137
13138
13139	/**
13140	* Handle a condition that occurred while delivering an event through the guest or
13141	* nested-guest IDT.
13142	*
13143	* @returns Strict VBox status code (i.e. informational status codes too).
13144	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
13145	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
13146	* to continue execution of the guest which will delivery the \#DF.
13147	* @retval VINF_EM_RESET if we detected a triple-fault condition.
13148	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
13149	*
13150	* @param pVCpu The cross context virtual CPU structure.
13151	* @param pVmxTransient The VMX-transient structure.
13152	*
13153	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13154	* Additionally, HMVMX_READ_EXIT_QUALIFICATION is required if the VM-exit
13155	* is due to an EPT violation, PML full or SPP-related event.
13156	*
13157	* @remarks No-long-jump zone!!!
13158	*/
13159	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13160	{
13161	Assert(!pVCpu->hm.s.Event.fPending);
13162	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
13163	if ( pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
13164	\|\| pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
13165	\|\| pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
13166	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_EXIT_QUALIFICATION);
13167
13168	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
13169	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13170	uint32_t const uIdtVectorInfo = pVmxTransient->uIdtVectoringInfo;
13171	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
13172	if (VMX_IDT_VECTORING_INFO_IS_VALID(uIdtVectorInfo))
13173	{
13174	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(uIdtVectorInfo);
13175	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(uIdtVectorInfo);
13176
13177	/*
13178	* If the event was a software interrupt (generated with INT n) or a software exception
13179	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
13180	* can handle the VM-exit and continue guest execution which will re-execute the
13181	* instruction rather than re-injecting the exception, as that can cause premature
13182	* trips to ring-3 before injection and involve TRPM which currently has no way of
13183	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
13184	* the problem).
13185	*/
13186	IEMXCPTRAISE enmRaise;
13187	IEMXCPTRAISEINFO fRaiseInfo;
13188	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
13189	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
13190	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
13191	{
13192	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
13193	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13194	}
13195	else if (VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo))
13196	{
13197	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
13198	uint8_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
13199	Assert(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT);
13200
13201	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
13202	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
13203
13204	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
13205
13206	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
13207	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
13208	{
13209	pVmxTransient->fVectoringPF = true;
13210	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13211	}
13212	}
13213	else
13214	{
13215	/*
13216	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
13217	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
13218	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
13219	*/
13220	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
13221	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13222	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
13223	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13224	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13225	}
13226
13227	/*
13228	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
13229	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
13230	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
13231	* subsequent VM-entry would fail, see @bugref{7445}.
13232	*
13233	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception".
13234	*/
13235	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13236	&& enmRaise == IEMXCPTRAISE_PREV_EVENT
13237	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
13238	&& CPUMIsGuestNmiBlocking(pVCpu))
13239	{
13240	CPUMSetGuestNmiBlocking(pVCpu, false);
13241	}
13242
13243	switch (enmRaise)
13244	{
13245	case IEMXCPTRAISE_CURRENT_XCPT:
13246	{
13247	Log4Func(("IDT: Pending secondary Xcpt: idtinfo=%#RX64 exitinfo=%#RX64\n", uIdtVectorInfo, uExitIntInfo));
13248	Assert(rcStrict == VINF_SUCCESS);
13249	break;
13250	}
13251
13252	case IEMXCPTRAISE_PREV_EVENT:
13253	{
13254	uint32_t u32ErrCode;
13255	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(uIdtVectorInfo))
13256	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
13257	else
13258	u32ErrCode = 0;
13259
13260	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
13261	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflect);
13262	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(uIdtVectorInfo), 0 /* cbInstr */,
13263	u32ErrCode, pVCpu->cpum.GstCtx.cr2);
13264
13265	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
13266	pVCpu->hm.s.Event.u32ErrCode));
13267	Assert(rcStrict == VINF_SUCCESS);
13268	break;
13269	}
13270
13271	case IEMXCPTRAISE_REEXEC_INSTR:
13272	Assert(rcStrict == VINF_SUCCESS);
13273	break;
13274
13275	case IEMXCPTRAISE_DOUBLE_FAULT:
13276	{
13277	/*
13278	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
13279	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
13280	*/
13281	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
13282	{
13283	pVmxTransient->fVectoringDoublePF = true;
13284	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
13285	pVCpu->cpum.GstCtx.cr2));
13286	rcStrict = VINF_SUCCESS;
13287	}
13288	else
13289	{
13290	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectConvertDF);
13291	hmR0VmxSetPendingXcptDF(pVCpu);
13292	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
13293	uIdtVector, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
13294	rcStrict = VINF_HM_DOUBLE_FAULT;
13295	}
13296	break;
13297	}
13298
13299	case IEMXCPTRAISE_TRIPLE_FAULT:
13300	{
13301	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector,
13302	VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
13303	rcStrict = VINF_EM_RESET;
13304	break;
13305	}
13306
13307	case IEMXCPTRAISE_CPU_HANG:
13308	{
13309	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
13310	rcStrict = VERR_EM_GUEST_CPU_HANG;
13311	break;
13312	}
13313
13314	default:
13315	{
13316	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
13317	rcStrict = VERR_VMX_IPE_2;
13318	break;
13319	}
13320	}
13321	}
13322	else if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
13323	&& !CPUMIsGuestNmiBlocking(pVCpu))
13324	{
13325	if ( VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo)
13326	&& VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo) != X86_XCPT_DF
13327	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(uExitIntInfo))
13328	{
13329	/*
13330	* Execution of IRET caused a fault when NMI blocking was in effect (i.e we're in
13331	* the guest or nested-guest NMI handler). We need to set the block-by-NMI field so
13332	* that NMIs remain blocked until the IRET execution is completed.
13333	*
13334	* See Intel spec. 31.7.1.2 "Resuming Guest Software After Handling An Exception".
13335	*/
13336	CPUMSetGuestNmiBlocking(pVCpu, true);
13337	Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
13338	}
13339	else if ( pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
13340	\|\| pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
13341	\|\| pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
13342	{
13343	/*
13344	* Execution of IRET caused an EPT violation, page-modification log-full event or
13345	* SPP-related event VM-exit when NMI blocking was in effect (i.e. we're in the
13346	* guest or nested-guest NMI handler). We need to set the block-by-NMI field so
13347	* that NMIs remain blocked until the IRET execution is completed.
13348	*
13349	* See Intel spec. 27.2.3 "Information about NMI unblocking due to IRET"
13350	*/
13351	if (VMX_EXIT_QUAL_EPT_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitQual))
13352	{
13353	CPUMSetGuestNmiBlocking(pVCpu, true);
13354	Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
13355	}
13356	}
13357	}
13358
13359	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
13360	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
13361	return rcStrict;
13362	}
13363
13364
13365	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13366	/**
13367	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
13368	* guest attempting to execute a VMX instruction.
13369	*
13370	* @returns Strict VBox status code (i.e. informational status codes too).
13371	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
13372	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
13373	*
13374	* @param pVCpu The cross context virtual CPU structure.
13375	* @param uExitReason The VM-exit reason.
13376	*
13377	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
13378	* @remarks No-long-jump zone!!!
13379	*/
13380	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
13381	{
13382	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
13383	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13384
13385	/*
13386	* The physical CPU would have already checked the CPU mode/code segment.
13387	* We shall just assert here for paranoia.
13388	* See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
13389	*/
13390	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
13391	Assert( !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
13392	\|\| CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));
13393
13394	if (uExitReason == VMX_EXIT_VMXON)
13395	{
13396	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
13397
13398	/*
13399	* We check CR4.VMXE because it is required to be always set while in VMX operation
13400	* by physical CPUs and our CR4 read-shadow is only consulted when executing specific
13401	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
13402	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
13403	*/
13404	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
13405	{
13406	Log4Func(("CR4.VMXE is not set -> #UD\n"));
13407	hmR0VmxSetPendingXcptUD(pVCpu);
13408	return VINF_HM_PENDING_XCPT;
13409	}
13410	}
13411	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
13412	{
13413	/*
13414	* The guest has not entered VMX operation but attempted to execute a VMX instruction
13415	* (other than VMXON), we need to raise a #UD.
13416	*/
13417	Log4Func(("Not in VMX root mode -> #UD\n"));
13418	hmR0VmxSetPendingXcptUD(pVCpu);
13419	return VINF_HM_PENDING_XCPT;
13420	}
13421
13422	/* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
13423	return VINF_SUCCESS;
13424	}
13425
13426	/**
13427	* Decodes the memory operand of an instruction that caused a VM-exit.
13428	*
13429	* The Exit qualification field provides the displacement field for memory
13430	* operand instructions, if any.
13431	*
13432	* @returns Strict VBox status code (i.e. informational status codes too).
13433	* @retval VINF_SUCCESS if the operand was successfully decoded.
13434	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
13435	* operand.
13436	* @param pVCpu The cross context virtual CPU structure.
13437	* @param uExitInstrInfo The VM-exit instruction information field.
13438	* @param enmMemAccess The memory operand's access type (read or write).
13439	* @param GCPtrDisp The instruction displacement field, if any. For
13440	* RIP-relative addressing pass RIP + displacement here.
13441	* @param pGCPtrMem Where to store the effective destination memory address.
13442	*
13443	* @remarks Warning! This function ASSUMES the instruction cannot be used in real or
13444	* virtual-8086 mode hence skips those checks while verifying if the
13445	* segment is valid.
13446	*/
13447	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
13448	PRTGCPTR pGCPtrMem)
13449	{
13450	Assert(pGCPtrMem);
13451	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
13452	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
13453	\| CPUMCTX_EXTRN_CR0);
13454
13455	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
13456	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
13457	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
13458
13459	VMXEXITINSTRINFO ExitInstrInfo;
13460	ExitInstrInfo.u = uExitInstrInfo;
13461	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
13462	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
13463	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
13464	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
13465	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
13466	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
13467	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
13468	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
13469	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
13470
13471	/*
13472	* Validate instruction information.
13473	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
13474	*/
13475	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
13476	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
13477	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
13478	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
13479	AssertLogRelMsgReturn(fIsMemOperand,
13480	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
13481
13482	/*
13483	* Compute the complete effective address.
13484	*
13485	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
13486	* See AMD spec. 4.5.2 "Segment Registers".
13487	*/
13488	RTGCPTR GCPtrMem = GCPtrDisp;
13489	if (fBaseRegValid)
13490	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
13491	if (fIdxRegValid)
13492	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
13493
13494	RTGCPTR const GCPtrOff = GCPtrMem;
13495	if ( !fIsLongMode
13496	\|\| iSegReg >= X86_SREG_FS)
13497	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
13498	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
13499
13500	/*
13501	* Validate effective address.
13502	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
13503	*/
13504	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
13505	Assert(cbAccess > 0);
13506	if (fIsLongMode)
13507	{
13508	if (X86_IS_CANONICAL(GCPtrMem))
13509	{
13510	*pGCPtrMem = GCPtrMem;
13511	return VINF_SUCCESS;
13512	}
13513
13514	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
13515	* "Data Limit Checks in 64-bit Mode". */
13516	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
13517	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13518	return VINF_HM_PENDING_XCPT;
13519	}
13520
13521	/*
13522	* This is a watered down version of iemMemApplySegment().
13523	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
13524	* and segment CPL/DPL checks are skipped.
13525	*/
13526	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
13527	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
13528	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
13529
13530	/* Check if the segment is present and usable. */
13531	if ( pSel->Attr.n.u1Present
13532	&& !pSel->Attr.n.u1Unusable)
13533	{
13534	Assert(pSel->Attr.n.u1DescType);
13535	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
13536	{
13537	/* Check permissions for the data segment. */
13538	if ( enmMemAccess == VMXMEMACCESS_WRITE
13539	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
13540	{
13541	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
13542	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
13543	return VINF_HM_PENDING_XCPT;
13544	}
13545
13546	/* Check limits if it's a normal data segment. */
13547	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
13548	{
13549	if ( GCPtrFirst32 > pSel->u32Limit
13550	\|\| GCPtrLast32 > pSel->u32Limit)
13551	{
13552	Log4Func(("Data segment limit exceeded. "
13553	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
13554	GCPtrLast32, pSel->u32Limit));
13555	if (iSegReg == X86_SREG_SS)
13556	hmR0VmxSetPendingXcptSS(pVCpu, 0);
13557	else
13558	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13559	return VINF_HM_PENDING_XCPT;
13560	}
13561	}
13562	else
13563	{
13564	/* Check limits if it's an expand-down data segment.
13565	Note! The upper boundary is defined by the B bit, not the G bit! */
13566	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
13567	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
13568	{
13569	Log4Func(("Expand-down data segment limit exceeded. "
13570	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
13571	GCPtrLast32, pSel->u32Limit));
13572	if (iSegReg == X86_SREG_SS)
13573	hmR0VmxSetPendingXcptSS(pVCpu, 0);
13574	else
13575	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13576	return VINF_HM_PENDING_XCPT;
13577	}
13578	}
13579	}
13580	else
13581	{
13582	/* Check permissions for the code segment. */
13583	if ( enmMemAccess == VMXMEMACCESS_WRITE
13584	\|\| ( enmMemAccess == VMXMEMACCESS_READ
13585	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
13586	{
13587	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
13588	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
13589	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13590	return VINF_HM_PENDING_XCPT;
13591	}
13592
13593	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
13594	if ( GCPtrFirst32 > pSel->u32Limit
13595	\|\| GCPtrLast32 > pSel->u32Limit)
13596	{
13597	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
13598	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
13599	if (iSegReg == X86_SREG_SS)
13600	hmR0VmxSetPendingXcptSS(pVCpu, 0);
13601	else
13602	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13603	return VINF_HM_PENDING_XCPT;
13604	}
13605	}
13606	}
13607	else
13608	{
13609	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
13610	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13611	return VINF_HM_PENDING_XCPT;
13612	}
13613
13614	*pGCPtrMem = GCPtrMem;
13615	return VINF_SUCCESS;
13616	}
13617	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13618
13619
13620	/**
13621	* VM-exit helper for LMSW.
13622	*/
13623	static VBOXSTRICTRC hmR0VmxExitLmsw(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst)
13624	{
13625	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13626	AssertRCReturn(rc, rc);
13627
13628	VBOXSTRICTRC rcStrict = IEMExecDecodedLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst);
13629	AssertMsg( rcStrict == VINF_SUCCESS
13630	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13631
13632	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13633	if (rcStrict == VINF_IEM_RAISED_XCPT)
13634	{
13635	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13636	rcStrict = VINF_SUCCESS;
13637	}
13638
13639	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
13640	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13641	return rcStrict;
13642	}
13643
13644
13645	/**
13646	* VM-exit helper for CLTS.
13647	*/
13648	static VBOXSTRICTRC hmR0VmxExitClts(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr)
13649	{
13650	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13651	AssertRCReturn(rc, rc);
13652
13653	VBOXSTRICTRC rcStrict = IEMExecDecodedClts(pVCpu, cbInstr);
13654	AssertMsg( rcStrict == VINF_SUCCESS
13655	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13656
13657	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13658	if (rcStrict == VINF_IEM_RAISED_XCPT)
13659	{
13660	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13661	rcStrict = VINF_SUCCESS;
13662	}
13663
13664	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
13665	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13666	return rcStrict;
13667	}
13668
13669
13670	/**
13671	* VM-exit helper for MOV from CRx (CRx read).
13672	*/
13673	static VBOXSTRICTRC hmR0VmxExitMovFromCrX(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
13674	{
13675	Assert(iCrReg < 16);
13676	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
13677
13678	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13679	AssertRCReturn(rc, rc);
13680
13681	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
13682	AssertMsg( rcStrict == VINF_SUCCESS
13683	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13684
13685	if (iGReg == X86_GREG_xSP)
13686	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
13687	else
13688	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13689	#ifdef VBOX_WITH_STATISTICS
13690	switch (iCrReg)
13691	{
13692	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
13693	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
13694	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
13695	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
13696	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
13697	}
13698	#endif
13699	Log4Func(("CR%d Read access rcStrict=%Rrc\n", iCrReg, VBOXSTRICTRC_VAL(rcStrict)));
13700	return rcStrict;
13701	}
13702
13703
13704	/**
13705	* VM-exit helper for MOV to CRx (CRx write).
13706	*/
13707	static VBOXSTRICTRC hmR0VmxExitMovToCrX(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
13708	{
13709	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13710	AssertRCReturn(rc, rc);
13711
13712	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
13713	AssertMsg( rcStrict == VINF_SUCCESS
13714	\|\| rcStrict == VINF_IEM_RAISED_XCPT
13715	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13716
13717	switch (iCrReg)
13718	{
13719	case 0:
13720	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13721	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
13722	Log4Func(("CR0 write. rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
13723	break;
13724
13725	case 2:
13726	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
13727	/* Nothing to do here, CR2 it's not part of the VMCS. */
13728	break;
13729
13730	case 3:
13731	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
13732	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
13733	Log4Func(("CR3 write. rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
13734	break;
13735
13736	case 4:
13737	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
13738	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
13739	Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
13740	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
13741	break;
13742
13743	case 8:
13744	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13745	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
13746	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
13747	break;
13748
13749	default:
13750	AssertMsgFailed(("Invalid CRx register %#x\n", iCrReg));
13751	break;
13752	}
13753
13754	if (rcStrict == VINF_IEM_RAISED_XCPT)
13755	{
13756	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13757	rcStrict = VINF_SUCCESS;
13758	}
13759	return rcStrict;
13760	}
13761
13762
13763	/**
13764	* VM-exit exception handler for \#PF (Page-fault exception).
13765	*
13766	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13767	*/
13768	static VBOXSTRICTRC hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13769	{
13770	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13771	PVM pVM = pVCpu->CTX_SUFF(pVM);
13772	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13773	AssertRCReturn(rc, rc);
13774
13775	if (!pVM->hm.s.fNestedPaging)
13776	{ /* likely */ }
13777	else
13778	{
13779	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13780	Assert(pVmxTransient->fIsNestedGuest \|\| pVCpu->hm.s.fUsingDebugLoop);
13781	#endif
13782	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13783	if (!pVmxTransient->fVectoringDoublePF)
13784	{
13785	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13786	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
13787	}
13788	else
13789	{
13790	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13791	Assert(!pVmxTransient->fIsNestedGuest);
13792	hmR0VmxSetPendingXcptDF(pVCpu);
13793	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13794	}
13795	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13796	return rc;
13797	}
13798
13799	Assert(!pVmxTransient->fIsNestedGuest);
13800
13801	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13802	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13803	if (pVmxTransient->fVectoringPF)
13804	{
13805	Assert(pVCpu->hm.s.Event.fPending);
13806	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13807	}
13808
13809	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13810	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13811	AssertRCReturn(rc, rc);
13812
13813	Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
13814	pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
13815
13816	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13817	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
13818
13819	Log4Func(("#PF: rc=%Rrc\n", rc));
13820	if (rc == VINF_SUCCESS)
13821	{
13822	/*
13823	* This is typically a shadow page table sync or a MMIO instruction. But we may have
13824	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13825	*/
13826	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13827	TRPMResetTrap(pVCpu);
13828	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13829	return rc;
13830	}
13831
13832	if (rc == VINF_EM_RAW_GUEST_TRAP)
13833	{
13834	if (!pVmxTransient->fVectoringDoublePF)
13835	{
13836	/* It's a guest page fault and needs to be reflected to the guest. */
13837	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13838	TRPMResetTrap(pVCpu);
13839	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13840	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13841	uGstErrorCode, pVmxTransient->uExitQual);
13842	}
13843	else
13844	{
13845	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13846	TRPMResetTrap(pVCpu);
13847	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13848	hmR0VmxSetPendingXcptDF(pVCpu);
13849	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13850	}
13851
13852	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13853	return VINF_SUCCESS;
13854	}
13855
13856	TRPMResetTrap(pVCpu);
13857	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13858	return rc;
13859	}
13860
13861
13862	/**
13863	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
13864	*
13865	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13866	*/
13867	static VBOXSTRICTRC hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13868	{
13869	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13870	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13871
13872	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
13873	AssertRCReturn(rc, rc);
13874
13875	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
13876	{
13877	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13878	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13879
13880	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13881	* provides VM-exit instruction length. If this causes problem later,
13882	* disassemble the instruction like it's done on AMD-V. */
13883	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13884	AssertRCReturn(rc2, rc2);
13885	return rc;
13886	}
13887
13888	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13889	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13890	return VINF_SUCCESS;
13891	}
13892
13893
13894	/**
13895	* VM-exit exception handler for \#BP (Breakpoint exception).
13896	*
13897	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13898	*/
13899	static VBOXSTRICTRC hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13900	{
13901	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13902	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13903
13904	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13905	AssertRCReturn(rc, rc);
13906
13907	if (!pVmxTransient->fIsNestedGuest)
13908	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx));
13909	else
13910	rc = VINF_EM_RAW_GUEST_TRAP;
13911	if (rc == VINF_EM_RAW_GUEST_TRAP)
13912	{
13913	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13914	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13915	}
13916
13917	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
13918	return rc;
13919	}
13920
13921
13922	/**
13923	* VM-exit exception handler for \#AC (Alignment-check exception).
13924	*
13925	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13926	*/
13927	static VBOXSTRICTRC hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13928	{
13929	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13930	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC);
13931
13932	/* Re-inject it. We'll detect any nesting before getting here. */
13933	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13934	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13935	return VINF_SUCCESS;
13936	}
13937
13938
13939	/**
13940	* VM-exit exception handler for \#DB (Debug exception).
13941	*
13942	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13943	*/
13944	static VBOXSTRICTRC hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13945	{
13946	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13947	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13948
13949	/*
13950	* Get the DR6-like values from the Exit qualification and pass it to DBGF for processing.
13951	*/
13952	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13953	AssertRCReturn(rc, rc);
13954
13955	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13956	uint64_t const uDR6 = X86_DR6_INIT_VAL
13957	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
13958	\| X86_DR6_BD \| X86_DR6_BS));
13959
13960	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13961	if (!pVmxTransient->fIsNestedGuest)
13962	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13963	else
13964	rc = VINF_EM_RAW_GUEST_TRAP;
13965	Log6Func(("rc=%Rrc\n", rc));
13966	if (rc == VINF_EM_RAW_GUEST_TRAP)
13967	{
13968	/*
13969	* The exception was for the guest. Update DR6, DR7.GD and
13970	* IA32_DEBUGCTL.LBR before forwarding it.
13971	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
13972	*/
13973	VMMRZCallRing3Disable(pVCpu);
13974	HM_DISABLE_PREEMPT(pVCpu);
13975
13976	pCtx->dr[6] &= ~X86_DR6_B_MASK;
13977	pCtx->dr[6] \|= uDR6;
13978	if (CPUMIsGuestDebugStateActive(pVCpu))
13979	ASMSetDR6(pCtx->dr[6]);
13980
13981	HM_RESTORE_PREEMPT();
13982	VMMRZCallRing3Enable(pVCpu);
13983
13984	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
13985	AssertRCReturn(rc, rc);
13986
13987	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13988	pCtx->dr[7] &= ~X86_DR7_GD;
13989
13990	/* Paranoia. */
13991	pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13992	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
13993
13994	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
13995	AssertRCReturn(rc, rc);
13996
13997	/*
13998	* Raise #DB in the guest.
13999	*
14000	* It is important to reflect exactly what the VM-exit gave us (preserving the
14001	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
14002	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
14003	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
14004	*
14005	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
14006	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
14007	*/
14008	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14009	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14010	return VINF_SUCCESS;
14011	}
14012
14013	/*
14014	* Not a guest trap, must be a hypervisor related debug event then.
14015	* Update DR6 in case someone is interested in it.
14016	*/
14017	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
14018	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
14019	CPUMSetHyperDR6(pVCpu, uDR6);
14020
14021	return rc;
14022	}
14023
14024
14025	/**
14026	* Hacks its way around the lovely mesa driver's backdoor accesses.
14027	*
14028	* @sa hmR0SvmHandleMesaDrvGp.
14029	*/
14030	static int hmR0VmxHandleMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14031	{
14032	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
14033	RT_NOREF(pCtx);
14034
14035	/* For now we'll just skip the instruction. */
14036	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14037	}
14038
14039
14040	/**
14041	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
14042	* backdoor logging w/o checking what it is running inside.
14043	*
14044	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
14045	* backdoor port and magic numbers loaded in registers.
14046	*
14047	* @returns true if it is, false if it isn't.
14048	* @sa hmR0SvmIsMesaDrvGp.
14049	*/
14050	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14051	{
14052	/* 0xed: IN eAX,dx */
14053	uint8_t abInstr[1];
14054	if (pVmxTransient->cbInstr != sizeof(abInstr))
14055	return false;
14056
14057	/* Check that it is #GP(0). */
14058	if (pVmxTransient->uExitIntErrorCode != 0)
14059	return false;
14060
14061	/* Check magic and port. */
14062	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
14063	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
14064	if (pCtx->rax != UINT32_C(0x564d5868))
14065	return false;
14066	if (pCtx->dx != UINT32_C(0x5658))
14067	return false;
14068
14069	/* Flat ring-3 CS. */
14070	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
14071	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
14072	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
14073	if (pCtx->cs.Attr.n.u2Dpl != 3)
14074	return false;
14075	if (pCtx->cs.u64Base != 0)
14076	return false;
14077
14078	/* Check opcode. */
14079	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
14080	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
14081	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
14082	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
14083	if (RT_FAILURE(rc))
14084	return false;
14085	if (abInstr[0] != 0xed)
14086	return false;
14087
14088	return true;
14089	}
14090
14091
14092	/**
14093	* VM-exit exception handler for \#GP (General-protection exception).
14094	*
14095	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
14096	*/
14097	static VBOXSTRICTRC hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14098	{
14099	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14100	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
14101
14102	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14103	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14104	if (pVmcsInfo->RealMode.fRealOnV86Active)
14105	{ /* likely */ }
14106	else
14107	{
14108	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14109	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv \|\| pVmxTransient->fIsNestedGuest);
14110	#endif
14111	/*
14112	* If the guest is not in real-mode or we have unrestricted guest execution support, or if we are
14113	* executing a nested-guest, reflect #GP to the guest or nested-guest.
14114	*/
14115	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14116	AssertRCReturn(rc, rc);
14117	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
14118	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
14119
14120	if ( pVmxTransient->fIsNestedGuest
14121	\|\| !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
14122	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
14123	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14124	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14125	else
14126	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
14127	return rc;
14128	}
14129
14130	Assert(CPUMIsGuestInRealModeEx(pCtx));
14131	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
14132	Assert(!pVmxTransient->fIsNestedGuest);
14133
14134	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14135	AssertRCReturn(rc, rc);
14136
14137	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
14138	if (rcStrict == VINF_SUCCESS)
14139	{
14140	if (!CPUMIsGuestInRealModeEx(pCtx))
14141	{
14142	/*
14143	* The guest is no longer in real-mode, check if we can continue executing the
14144	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
14145	*/
14146	pVmcsInfo->RealMode.fRealOnV86Active = false;
14147	if (HMCanExecuteVmxGuest(pVCpu, pCtx))
14148	{
14149	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
14150	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14151	}
14152	else
14153	{
14154	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
14155	rcStrict = VINF_EM_RESCHEDULE;
14156	}
14157	}
14158	else
14159	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14160	}
14161	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14162	{
14163	rcStrict = VINF_SUCCESS;
14164	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14165	}
14166	return VBOXSTRICTRC_VAL(rcStrict);
14167	}
14168
14169
14170	/**
14171	* VM-exit exception handler wrapper for all other exceptions that are not handled
14172	* by a specific handler.
14173	*
14174	* This simply re-injects the exception back into the VM without any special
14175	* processing.
14176	*
14177	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
14178	*/
14179	static VBOXSTRICTRC hmR0VmxExitXcptOthers(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14180	{
14181	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14182
14183	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14184	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14185	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active \|\| pVmxTransient->fIsNestedGuest,
14186	("uVector=%#x u32XcptBitmap=%#X32\n",
14187	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
14188	NOREF(pVmcsInfo);
14189	#endif
14190
14191	/*
14192	* Re-inject the exception into the guest. This cannot be a double-fault condition which
14193	* would have been handled while checking exits due to event delivery.
14194	*/
14195	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
14196
14197	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14198	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
14199	AssertRCReturn(rc, rc);
14200	Log4Func(("Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
14201	#endif
14202
14203	#ifdef VBOX_WITH_STATISTICS
14204	switch (uVector)
14205	{
14206	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE); break;
14207	case X86_XCPT_DB: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB); break;
14208	case X86_XCPT_BP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP); break;
14209	case X86_XCPT_OF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
14210	case X86_XCPT_BR: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBR); break;
14211	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD); break;
14212	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
14213	case X86_XCPT_DF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDF); break;
14214	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS); break;
14215	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP); break;
14216	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS); break;
14217	case X86_XCPT_GP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP); break;
14218	case X86_XCPT_PF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF); break;
14219	case X86_XCPT_MF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF); break;
14220	case X86_XCPT_AC: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC); break;
14221	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF); break;
14222	default:
14223	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
14224	break;
14225	}
14226	#endif
14227
14228	/* We should never call this function for a page-fault, we'd need to pass on the fault address below otherwise. */
14229	Assert(!VMX_EXIT_INT_INFO_IS_XCPT_PF(pVmxTransient->uExitIntInfo));
14230	NOREF(uVector);
14231
14232	/* Re-inject the original exception into the guest. */
14233	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14234	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14235	return VINF_SUCCESS;
14236	}
14237
14238
14239	/**
14240	* VM-exit exception handler for all exceptions (except NMIs!).
14241	*
14242	* @remarks This may be called for both guests and nested-guests. Take care to not
14243	* make assumptions and avoid doing anything that is not relevant when
14244	* executing a nested-guest (e.g., Mesa driver hacks).
14245	*/
14246	static VBOXSTRICTRC hmR0VmxExitXcpt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14247	{
14248	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
14249
14250	/*
14251	* If this VM-exit occurred while delivering an event through the guest IDT, take
14252	* action based on the return code and additional hints (e.g. for page-faults)
14253	* that will be updated in the VMX transient structure.
14254	*/
14255	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14256	if (rcStrict == VINF_SUCCESS)
14257	{
14258	/*
14259	* If an exception caused a VM-exit due to delivery of an event, the original
14260	* event may have to be re-injected into the guest. We shall reinject it and
14261	* continue guest execution. However, page-fault is a complicated case and
14262	* needs additional processing done in hmR0VmxExitXcptPF().
14263	*/
14264	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
14265	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
14266	if ( !pVCpu->hm.s.Event.fPending
14267	\|\| uVector == X86_XCPT_PF)
14268	{
14269	switch (uVector)
14270	{
14271	case X86_XCPT_PF: return hmR0VmxExitXcptPF(pVCpu, pVmxTransient);
14272	case X86_XCPT_GP: return hmR0VmxExitXcptGP(pVCpu, pVmxTransient);
14273	case X86_XCPT_MF: return hmR0VmxExitXcptMF(pVCpu, pVmxTransient);
14274	case X86_XCPT_DB: return hmR0VmxExitXcptDB(pVCpu, pVmxTransient);
14275	case X86_XCPT_BP: return hmR0VmxExitXcptBP(pVCpu, pVmxTransient);
14276	case X86_XCPT_AC: return hmR0VmxExitXcptAC(pVCpu, pVmxTransient);
14277	default:
14278	return hmR0VmxExitXcptOthers(pVCpu, pVmxTransient);
14279	}
14280	}
14281	/* else: inject pending event before resuming guest execution. */
14282	}
14283	else if (rcStrict == VINF_HM_DOUBLE_FAULT)
14284	{
14285	Assert(pVCpu->hm.s.Event.fPending);
14286	rcStrict = VINF_SUCCESS;
14287	}
14288
14289	return rcStrict;
14290	}
14291	/** @} */
14292
14293
14294	/** @name VM-exit handlers.
14295	* @{
14296	*/
14297	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14298	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
14299	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14300
14301	/**
14302	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
14303	*/
14304	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14305	{
14306	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14307	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
14308	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
14309	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
14310	return VINF_SUCCESS;
14311	return VINF_EM_RAW_INTERRUPT;
14312	}
14313
14314
14315	/**
14316	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI). Conditional
14317	* VM-exit.
14318	*/
14319	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14320	{
14321	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14322	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
14323
14324	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14325	AssertRCReturn(rc, rc);
14326
14327	uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
14328	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
14329	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
14330
14331	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14332	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
14333	&& uExitIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
14334	NOREF(pVmcsInfo);
14335
14336	VBOXSTRICTRC rcStrict;
14337	switch (uExitIntType)
14338	{
14339	/*
14340	* Host physical NMIs:
14341	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
14342	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
14343	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
14344	*
14345	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
14346	* See Intel spec. 27.5.5 "Updating Non-Register State".
14347	*/
14348	case VMX_EXIT_INT_INFO_TYPE_NMI:
14349	{
14350	rcStrict = hmR0VmxExitHostNmi(pVCpu, pVmcsInfo);
14351	break;
14352	}
14353
14354	/*
14355	* Privileged software exceptions (#DB from ICEBP),
14356	* Software exceptions (#BP and #OF),
14357	* Hardware exceptions:
14358	* Process the required exceptions and resume guest execution if possible.
14359	*/
14360	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
14361	Assert(uVector == X86_XCPT_DB);
14362	RT_FALL_THRU();
14363	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
14364	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uExitIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
14365	RT_FALL_THRU();
14366	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
14367	{
14368	NOREF(uVector);
14369	rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14370	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14371	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
14372	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
14373	AssertRCReturn(rc, rc);
14374
14375	rcStrict = hmR0VmxExitXcpt(pVCpu, pVmxTransient);
14376	break;
14377	}
14378
14379	default:
14380	{
14381	pVCpu->hm.s.u32HMError = pVmxTransient->uExitIntInfo;
14382	rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
14383	AssertMsgFailed(("Invalid/unexpected VM-exit interruption info %#x\n", pVmxTransient->uExitIntInfo));
14384	break;
14385	}
14386	}
14387
14388	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
14389	return rcStrict;
14390	}
14391
14392
14393	/**
14394	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
14395	*/
14396	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14397	{
14398	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14399
14400	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
14401	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14402	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
14403	AssertRCReturn(rc, rc);
14404
14405	/* Evaluate and deliver pending events and resume guest execution. */
14406	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
14407	return VINF_SUCCESS;
14408	}
14409
14410
14411	/**
14412	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
14413	*/
14414	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14415	{
14416	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14417
14418	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14419	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
14420	{
14421	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
14422	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14423	}
14424
14425	Assert(!CPUMIsGuestNmiBlocking(pVCpu));
14426
14427	/*
14428	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
14429	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
14430	*/
14431	uint32_t fIntrState;
14432	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
14433	AssertRCReturn(rc, rc);
14434	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
14435	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
14436	{
14437	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
14438	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
14439
14440	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
14441	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
14442	AssertRCReturn(rc, rc);
14443	}
14444
14445	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
14446	rc = hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
14447	AssertRCReturn(rc, rc);
14448
14449	/* Evaluate and deliver pending events and resume guest execution. */
14450	return VINF_SUCCESS;
14451	}
14452
14453
14454	/**
14455	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
14456	*/
14457	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14458	{
14459	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14460	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14461	}
14462
14463
14464	/**
14465	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
14466	*/
14467	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14468	{
14469	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14470	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14471	}
14472
14473
14474	/**
14475	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
14476	*/
14477	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14478	{
14479	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14480
14481	/*
14482	* Get the state we need and update the exit history entry.
14483	*/
14484	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14485	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14486	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
14487	AssertRCReturn(rc, rc);
14488
14489	VBOXSTRICTRC rcStrict;
14490	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14491	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
14492	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14493	if (!pExitRec)
14494	{
14495	/*
14496	* Regular CPUID instruction execution.
14497	*/
14498	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
14499	if (rcStrict == VINF_SUCCESS)
14500	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14501	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14502	{
14503	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14504	rcStrict = VINF_SUCCESS;
14505	}
14506	}
14507	else
14508	{
14509	/*
14510	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14511	*/
14512	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14513	AssertRCReturn(rc2, rc2);
14514
14515	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
14516	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
14517
14518	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14519	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14520
14521	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14522	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14523	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14524	}
14525	return rcStrict;
14526	}
14527
14528
14529	/**
14530	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
14531	*/
14532	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14533	{
14534	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14535
14536	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14537	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
14538	AssertRCReturn(rc, rc);
14539
14540	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
14541	return VINF_EM_RAW_EMULATE_INSTR;
14542
14543	AssertMsgFailed(("hmR0VmxExitGetsec: Unexpected VM-exit when CR4.SMXE is 0.\n"));
14544	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14545	}
14546
14547
14548	/**
14549	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
14550	*/
14551	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14552	{
14553	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14554
14555	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14556	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14557	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14558	AssertRCReturn(rc, rc);
14559
14560	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
14561	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14562	{
14563	/* If we get a spurious VM-exit when TSC offsetting is enabled,
14564	we must reset offsetting on VM-entry. See @bugref{6634}. */
14565	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
14566	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14567	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14568	}
14569	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14570	{
14571	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14572	rcStrict = VINF_SUCCESS;
14573	}
14574	return rcStrict;
14575	}
14576
14577
14578	/**
14579	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
14580	*/
14581	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14582	{
14583	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14584
14585	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14586	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
14587	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14588	AssertRCReturn(rc, rc);
14589
14590	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
14591	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14592	{
14593	/* If we get a spurious VM-exit when TSC offsetting is enabled,
14594	we must reset offsetting on VM-reentry. See @bugref{6634}. */
14595	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
14596	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14597	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14598	}
14599	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14600	{
14601	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14602	rcStrict = VINF_SUCCESS;
14603	}
14604	return rcStrict;
14605	}
14606
14607
14608	/**
14609	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
14610	*/
14611	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14612	{
14613	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14614
14615	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14616	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
14617	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
14618	AssertRCReturn(rc, rc);
14619
14620	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14621	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
14622	if (RT_LIKELY(rc == VINF_SUCCESS))
14623	{
14624	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14625	Assert(pVmxTransient->cbInstr == 2);
14626	}
14627	else
14628	{
14629	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
14630	rc = VERR_EM_INTERPRETER;
14631	}
14632	return rc;
14633	}
14634
14635
14636	/**
14637	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
14638	*/
14639	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14640	{
14641	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14642
14643	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
14644	if (EMAreHypercallInstructionsEnabled(pVCpu))
14645	{
14646	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14647	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
14648	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
14649	AssertRCReturn(rc, rc);
14650
14651	/* Perform the hypercall. */
14652	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
14653	if (rcStrict == VINF_SUCCESS)
14654	{
14655	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14656	AssertRCReturn(rc, rc);
14657	}
14658	else
14659	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
14660	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
14661	\|\| RT_FAILURE(rcStrict));
14662
14663	/* If the hypercall changes anything other than guest's general-purpose registers,
14664	we would need to reload the guest changed bits here before VM-entry. */
14665	}
14666	else
14667	Log4Func(("Hypercalls not enabled\n"));
14668
14669	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
14670	if (RT_FAILURE(rcStrict))
14671	{
14672	hmR0VmxSetPendingXcptUD(pVCpu);
14673	rcStrict = VINF_SUCCESS;
14674	}
14675
14676	return rcStrict;
14677	}
14678
14679
14680	/**
14681	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
14682	*/
14683	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14684	{
14685	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14686	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
14687
14688	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14689	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14690	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14691	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
14692	AssertRCReturn(rc, rc);
14693
14694	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
14695
14696	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
14697	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14698	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14699	{
14700	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14701	rcStrict = VINF_SUCCESS;
14702	}
14703	else
14704	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) status: %Rrc\n", pVmxTransient->uExitQual,
14705	VBOXSTRICTRC_VAL(rcStrict)));
14706	return rcStrict;
14707	}
14708
14709
14710	/**
14711	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
14712	*/
14713	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14714	{
14715	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14716
14717	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14718	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
14719	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14720	AssertRCReturn(rc, rc);
14721
14722	VBOXSTRICTRC rcStrict = IEMExecDecodedMonitor(pVCpu, pVmxTransient->cbInstr);
14723	if (rcStrict == VINF_SUCCESS)
14724	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14725	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14726	{
14727	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14728	rcStrict = VINF_SUCCESS;
14729	}
14730
14731	return rcStrict;
14732	}
14733
14734
14735	/**
14736	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
14737	*/
14738	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14739	{
14740	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14741
14742	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14743	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
14744	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14745	AssertRCReturn(rc, rc);
14746
14747	VBOXSTRICTRC rcStrict = IEMExecDecodedMwait(pVCpu, pVmxTransient->cbInstr);
14748	if (RT_SUCCESS(rcStrict))
14749	{
14750	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14751	if (EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
14752	rcStrict = VINF_SUCCESS;
14753	}
14754
14755	return rcStrict;
14756	}
14757
14758
14759	/**
14760	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
14761	* VM-exit.
14762	*/
14763	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14764	{
14765	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14766	return VINF_EM_RESET;
14767	}
14768
14769
14770	/**
14771	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
14772	*/
14773	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14774	{
14775	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14776
14777	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14778	AssertRCReturn(rc, rc);
14779
14780	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
14781	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
14782	rc = VINF_SUCCESS;
14783	else
14784	rc = VINF_EM_HALT;
14785
14786	if (rc != VINF_SUCCESS)
14787	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
14788	return rc;
14789	}
14790
14791
14792	/**
14793	* VM-exit handler for instructions that result in a \#UD exception delivered to
14794	* the guest.
14795	*/
14796	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14797	{
14798	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14799	hmR0VmxSetPendingXcptUD(pVCpu);
14800	return VINF_SUCCESS;
14801	}
14802
14803
14804	/**
14805	* VM-exit handler for expiry of the VMX-preemption timer.
14806	*/
14807	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14808	{
14809	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14810
14811	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
14812	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14813
14814	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
14815	PVM pVM = pVCpu->CTX_SUFF(pVM);
14816	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
14817	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
14818	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
14819	}
14820
14821
14822	/**
14823	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
14824	*/
14825	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14826	{
14827	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14828
14829	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14830	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14831	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
14832	AssertRCReturn(rc, rc);
14833
14834	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
14835	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
14836	: HM_CHANGED_RAISED_XCPT_MASK);
14837
14838	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14839	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
14840
14841	return rcStrict;
14842	}
14843
14844
14845	/**
14846	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
14847	*/
14848	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14849	{
14850	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14851	/** @todo Use VM-exit instruction information. */
14852	return VERR_EM_INTERPRETER;
14853	}
14854
14855
14856	/**
14857	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error
14858	* VM-exit.
14859	*/
14860	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14861	{
14862	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14863	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14864	AssertRCReturn(rc, rc);
14865
14866	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
14867	if (RT_FAILURE(rc))
14868	return rc;
14869
14870	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
14871	NOREF(uInvalidReason);
14872
14873	#ifdef VBOX_STRICT
14874	uint32_t fIntrState;
14875	RTHCUINTREG uHCReg;
14876	uint64_t u64Val;
14877	uint32_t u32Val;
14878	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
14879	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
14880	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
14881	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
14882	AssertRCReturn(rc, rc);
14883
14884	Log4(("uInvalidReason %u\n", uInvalidReason));
14885	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
14886	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
14887	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
14888	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
14889
14890	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
14891	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
14892	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
14893	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
14894	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
14895	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
14896	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
14897	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
14898	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
14899	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
14900	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
14901	{
14902	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
14903	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
14904	}
14905
14906	hmR0DumpRegs(pVCpu);
14907	#endif
14908
14909	return VERR_VMX_INVALID_GUEST_STATE;
14910	}
14911
14912	/**
14913	* VM-exit handler for all undefined/unexpected reasons. Should never happen.
14914	*/
14915	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUnexpected(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14916	{
14917	/*
14918	* Cummulative notes of all recognized but unexpected VM-exits.
14919	*
14920	* 1. This does -not- cover scenarios like like a page-fault VM-exit occurring when
14921	* nested-paging is used.
14922	*
14923	* 2. Any instruction that causes a VM-exit unconditionally (for e.g. VMXON) must be
14924	* emulated or a #UD must be raised in the guest. Therefore, we should -not- be using
14925	* this function (and thereby stop VM execution) for handling such instructions.
14926	*
14927	*
14928	* VMX_EXIT_INIT_SIGNAL:
14929	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
14930	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these
14931	* VM-exits. However, we should not receive INIT signals VM-exit while executing a VM.
14932	*
14933	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery"
14934	* See Intel spec. 29.3 "VMX Instructions" for "VMXON".
14935	* See Intel spec. "23.8 Restrictions on VMX operation".
14936	*
14937	* VMX_EXIT_SIPI:
14938	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest
14939	* activity state is used. We don't make use of it as our guests don't have direct
14940	* access to the host local APIC.
14941	*
14942	* See Intel spec. 25.3 "Other Causes of VM-exits".
14943	*
14944	* VMX_EXIT_IO_SMI:
14945	* VMX_EXIT_SMI:
14946	* This can only happen if we support dual-monitor treatment of SMI, which can be
14947	* activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer
14948	* monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in
14949	* VMX root mode or receive an SMI. If we get here, something funny is going on.
14950	*
14951	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
14952	* See Intel spec. 25.3 "Other Causes of VM-Exits"
14953	*
14954	* VMX_EXIT_ERR_MSR_LOAD:
14955	* Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected
14956	* and typically indicates a bug in the hypervisor code. We thus cannot not resume
14957	* execution.
14958	*
14959	* See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State".
14960	*
14961	* VMX_EXIT_ERR_MACHINE_CHECK:
14962	* Machine check exceptions indicates a fatal/unrecoverable hardware condition
14963	* including but not limited to system bus, ECC, parity, cache and TLB errors. A
14964	* #MC exception abort class exception is raised. We thus cannot assume a
14965	* reasonable chance of continuing any sort of execution and we bail.
14966	*
14967	* See Intel spec. 15.1 "Machine-check Architecture".
14968	* See Intel spec. 27.1 "Architectural State Before A VM Exit".
14969	*
14970	* VMX_EXIT_PML_FULL:
14971	* VMX_EXIT_VIRTUALIZED_EOI:
14972	* VMX_EXIT_APIC_WRITE:
14973	* We do not currently support any of these features and thus they are all unexpected
14974	* VM-exits.
14975	*
14976	* VMX_EXIT_GDTR_IDTR_ACCESS:
14977	* VMX_EXIT_LDTR_TR_ACCESS:
14978	* VMX_EXIT_RDRAND:
14979	* VMX_EXIT_RSM:
14980	* VMX_EXIT_VMFUNC:
14981	* VMX_EXIT_ENCLS:
14982	* VMX_EXIT_RDSEED:
14983	* VMX_EXIT_XSAVES:
14984	* VMX_EXIT_XRSTORS:
14985	* VMX_EXIT_UMWAIT:
14986	* VMX_EXIT_TPAUSE:
14987	* These VM-exits are -not- caused unconditionally by execution of the corresponding
14988	* instruction. Any VM-exit for these instructions indicate a hardware problem,
14989	* unsupported CPU modes (like SMM) or potentially corrupt VMCS controls.
14990	*
14991	* See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
14992	*/
14993	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14994	AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason));
14995	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14996	}
14997
14998
14999	/**
15000	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
15001	*/
15002	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15003	{
15004	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15005
15006	/** @todo Optimize this: We currently drag in in the whole MSR state
15007	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
15008	* MSRs required. That would require changes to IEM and possibly CPUM too.
15009	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
15010	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15011	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
15012	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
15013	switch (idMsr)
15014	{
15015	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
15016	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
15017	}
15018
15019	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15020	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
15021	AssertRCReturn(rc, rc);
15022
15023	Log4Func(("ecx=%#RX32\n", idMsr));
15024
15025	#ifdef VBOX_STRICT
15026	Assert(!pVmxTransient->fIsNestedGuest);
15027	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
15028	{
15029	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
15030	&& idMsr != MSR_K6_EFER)
15031	{
15032	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
15033	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15034	}
15035	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
15036	{
15037	Assert(pVmcsInfo->pvMsrBitmap);
15038	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
15039	if (fMsrpm & VMXMSRPM_ALLOW_RD)
15040	{
15041	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
15042	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15043	}
15044	}
15045	}
15046	#endif
15047
15048	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
15049	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
15050	if (rcStrict == VINF_SUCCESS)
15051	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
15052	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
15053	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15054	{
15055	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15056	rcStrict = VINF_SUCCESS;
15057	}
15058	else
15059	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
15060
15061	return rcStrict;
15062	}
15063
15064
15065	/**
15066	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
15067	*/
15068	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15069	{
15070	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15071
15072	/** @todo Optimize this: We currently drag in in the whole MSR state
15073	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
15074	* MSRs required. That would require changes to IEM and possibly CPUM too.
15075	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
15076	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
15077	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
15078
15079	/*
15080	* The FS and GS base MSRs are not part of the above all-MSRs mask.
15081	* Although we don't need to fetch the base as it will be overwritten shortly, while
15082	* loading guest-state we would also load the entire segment register including limit
15083	* and attributes and thus we need to load them here.
15084	*/
15085	switch (idMsr)
15086	{
15087	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
15088	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
15089	}
15090
15091	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15092	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15093	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
15094	AssertRCReturn(rc, rc);
15095
15096	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
15097
15098	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
15099	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
15100
15101	if (rcStrict == VINF_SUCCESS)
15102	{
15103	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15104
15105	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
15106	if ( idMsr == MSR_IA32_APICBASE
15107	\|\| ( idMsr >= MSR_IA32_X2APIC_START
15108	&& idMsr <= MSR_IA32_X2APIC_END))
15109	{
15110	/*
15111	* We've already saved the APIC related guest-state (TPR) in post-run phase.
15112	* When full APIC register virtualization is implemented we'll have to make
15113	* sure APIC state is saved from the VMCS before IEM changes it.
15114	*/
15115	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
15116	}
15117	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
15118	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
15119	else if (idMsr == MSR_K6_EFER)
15120	{
15121	/*
15122	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
15123	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
15124	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
15125	*/
15126	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
15127	}
15128
15129	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not used. */
15130	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
15131	{
15132	switch (idMsr)
15133	{
15134	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
15135	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
15136	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
15137	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
15138	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
15139	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
15140	default:
15141	{
15142	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
15143	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
15144	else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
15145	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
15146	break;
15147	}
15148	}
15149	}
15150	#ifdef VBOX_STRICT
15151	else
15152	{
15153	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
15154	switch (idMsr)
15155	{
15156	case MSR_IA32_SYSENTER_CS:
15157	case MSR_IA32_SYSENTER_EIP:
15158	case MSR_IA32_SYSENTER_ESP:
15159	case MSR_K8_FS_BASE:
15160	case MSR_K8_GS_BASE:
15161	{
15162	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
15163	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15164	}
15165
15166	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
15167	default:
15168	{
15169	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
15170	{
15171	/* EFER MSR writes are always intercepted. */
15172	if (idMsr != MSR_K6_EFER)
15173	{
15174	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
15175	idMsr));
15176	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15177	}
15178	}
15179
15180	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
15181	{
15182	Assert(pVmcsInfo->pvMsrBitmap);
15183	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
15184	if (fMsrpm & VMXMSRPM_ALLOW_WR)
15185	{
15186	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
15187	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15188	}
15189	}
15190	break;
15191	}
15192	}
15193	}
15194	#endif /* VBOX_STRICT */
15195	}
15196	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15197	{
15198	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15199	rcStrict = VINF_SUCCESS;
15200	}
15201	else
15202	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
15203
15204	return rcStrict;
15205	}
15206
15207
15208	/**
15209	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
15210	*/
15211	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15212	{
15213	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15214
15215	/** @todo The guest has likely hit a contended spinlock. We might want to
15216	* poke a schedule different guest VCPU. */
15217	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15218	if (RT_SUCCESS(rc))
15219	return VINF_EM_RAW_INTERRUPT;
15220
15221	AssertMsgFailed(("hmR0VmxExitPause: Failed to increment RIP. rc=%Rrc\n", rc));
15222	return rc;
15223	}
15224
15225
15226	/**
15227	* VM-exit handler for when the TPR value is lowered below the specified
15228	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
15229	*/
15230	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15231	{
15232	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15233	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
15234
15235	/*
15236	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
15237	* We'll re-evaluate pending interrupts and inject them before the next VM
15238	* entry so we can just continue execution here.
15239	*/
15240	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
15241	return VINF_SUCCESS;
15242	}
15243
15244
15245	/**
15246	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
15247	* VM-exit.
15248	*
15249	* @retval VINF_SUCCESS when guest execution can continue.
15250	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
15251	* @retval VERR_EM_RESCHEDULE_REM when we need to return to ring-3 due to
15252	* incompatible guest state for VMX execution (real-on-v86 case).
15253	*/
15254	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15255	{
15256	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15257	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
15258
15259	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15260	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15261	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15262	AssertRCReturn(rc, rc);
15263
15264	VBOXSTRICTRC rcStrict;
15265	PVM pVM = pVCpu->CTX_SUFF(pVM);
15266	uint64_t const uExitQual = pVmxTransient->uExitQual;
15267	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
15268	switch (uAccessType)
15269	{
15270	/*
15271	* MOV to CRx.
15272	*/
15273	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
15274	{
15275	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15276	AssertRCReturn(rc, rc);
15277
15278	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
15279	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
15280	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
15281	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
15282
15283	/*
15284	* MOV to CR3 only cause a VM-exit when one or more of the following are true:
15285	* - When nested paging isn't used.
15286	* - If the guest doesn't have paging enabled (intercept CR3 to update shadow page tables).
15287	* - We are executing in the VM debug loop.
15288	*/
15289	Assert( iCrReg != 3
15290	\|\| !pVM->hm.s.fNestedPaging
15291	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
15292	\|\| pVCpu->hm.s.fUsingDebugLoop);
15293
15294	/* MOV to CR8 writes only cause VM-exits when TPR shadow is not used. */
15295	Assert( iCrReg != 8
15296	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
15297
15298	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
15299	AssertMsg( rcStrict == VINF_SUCCESS
15300	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15301
15302	/*
15303	* This is a kludge for handling switches back to real mode when we try to use
15304	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
15305	* deal with special selector values, so we have to return to ring-3 and run
15306	* there till the selector values are V86 mode compatible.
15307	*
15308	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
15309	* latter is an alias for VINF_IEM_RAISED_XCPT which is asserted at the end of
15310	* this function.
15311	*/
15312	if ( iCrReg == 0
15313	&& rcStrict == VINF_SUCCESS
15314	&& !pVM->hm.s.vmx.fUnrestrictedGuest
15315	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
15316	&& (uOldCr0 & X86_CR0_PE)
15317	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
15318	{
15319	/** @todo Check selectors rather than returning all the time. */
15320	Assert(!pVmxTransient->fIsNestedGuest);
15321	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
15322	rcStrict = VINF_EM_RESCHEDULE_REM;
15323	}
15324	break;
15325	}
15326
15327	/*
15328	* MOV from CRx.
15329	*/
15330	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
15331	{
15332	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
15333	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
15334
15335	/*
15336	* MOV from CR3 only cause a VM-exit when one or more of the following are true:
15337	* - When nested paging isn't used.
15338	* - If the guest doesn't have paging enabled (pass guest's CR3 rather than our identity mapped CR3).
15339	* - We are executing in the VM debug loop.
15340	*/
15341	Assert( iCrReg != 3
15342	\|\| !pVM->hm.s.fNestedPaging
15343	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
15344	\|\| pVCpu->hm.s.fUsingDebugLoop);
15345
15346	/* MOV from CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
15347	Assert( iCrReg != 8
15348	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
15349
15350	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
15351	break;
15352	}
15353
15354	/*
15355	* CLTS (Clear Task-Switch Flag in CR0).
15356	*/
15357	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
15358	{
15359	rcStrict = hmR0VmxExitClts(pVCpu, pVmcsInfo, pVmxTransient->cbInstr);
15360	break;
15361	}
15362
15363	/*
15364	* LMSW (Load Machine-Status Word into CR0).
15365	* LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here.
15366	*/
15367	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW:
15368	{
15369	RTGCPTR GCPtrEffDst;
15370	uint8_t const cbInstr = pVmxTransient->cbInstr;
15371	uint16_t const uMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual);
15372	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(uExitQual);
15373	if (fMemOperand)
15374	{
15375	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
15376	AssertRCReturn(rc, rc);
15377	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
15378	}
15379	else
15380	GCPtrEffDst = NIL_RTGCPTR;
15381	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmcsInfo, cbInstr, uMsw, GCPtrEffDst);
15382	break;
15383	}
15384
15385	default:
15386	{
15387	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
15388	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
15389	}
15390	}
15391
15392	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
15393	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
15394	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15395
15396	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
15397	NOREF(pVM);
15398	return rcStrict;
15399	}
15400
15401
15402	/**
15403	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
15404	* VM-exit.
15405	*/
15406	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15407	{
15408	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15409	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
15410
15411	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15412	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15413	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15414	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15415	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
15416	\| CPUMCTX_EXTRN_EFER);
15417	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
15418	AssertRCReturn(rc, rc);
15419
15420	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
15421	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
15422	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
15423	bool const fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
15424	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
15425	bool const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
15426	bool const fDbgStepping = pVCpu->hm.s.fSingleInstruction;
15427	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
15428
15429	/*
15430	* Update exit history to see if this exit can be optimized.
15431	*/
15432	VBOXSTRICTRC rcStrict;
15433	PCEMEXITREC pExitRec = NULL;
15434	if ( !fGstStepping
15435	&& !fDbgStepping)
15436	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
15437	!fIOString
15438	? !fIOWrite
15439	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
15440	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
15441	: !fIOWrite
15442	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
15443	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
15444	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
15445	if (!pExitRec)
15446	{
15447	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
15448	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
15449
15450	uint32_t const cbValue = s_aIOSizes[uIOSize];
15451	uint32_t const cbInstr = pVmxTransient->cbInstr;
15452	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
15453	PVM pVM = pVCpu->CTX_SUFF(pVM);
15454	if (fIOString)
15455	{
15456	/*
15457	* INS/OUTS - I/O String instruction.
15458	*
15459	* Use instruction-information if available, otherwise fall back on
15460	* interpreting the instruction.
15461	*/
15462	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
15463	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
15464	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
15465	if (fInsOutsInfo)
15466	{
15467	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15468	AssertRCReturn(rc2, rc2);
15469	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
15470	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
15471	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
15472	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
15473	if (fIOWrite)
15474	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
15475	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
15476	else
15477	{
15478	/*
15479	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
15480	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
15481	* See Intel Instruction spec. for "INS".
15482	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
15483	*/
15484	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
15485	}
15486	}
15487	else
15488	rcStrict = IEMExecOne(pVCpu);
15489
15490	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
15491	fUpdateRipAlready = true;
15492	}
15493	else
15494	{
15495	/*
15496	* IN/OUT - I/O instruction.
15497	*/
15498	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
15499	uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
15500	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
15501	if (fIOWrite)
15502	{
15503	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
15504	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
15505	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
15506	&& !pCtx->eflags.Bits.u1TF)
15507	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
15508	}
15509	else
15510	{
15511	uint32_t u32Result = 0;
15512	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
15513	if (IOM_SUCCESS(rcStrict))
15514	{
15515	/* Save result of I/O IN instr. in AL/AX/EAX. */
15516	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
15517	}
15518	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
15519	&& !pCtx->eflags.Bits.u1TF)
15520	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
15521	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
15522	}
15523	}
15524
15525	if (IOM_SUCCESS(rcStrict))
15526	{
15527	if (!fUpdateRipAlready)
15528	{
15529	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
15530	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
15531	}
15532
15533	/*
15534	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
15535	* while booting Fedora 17 64-bit guest.
15536	*
15537	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
15538	*/
15539	if (fIOString)
15540	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
15541
15542	/*
15543	* If any I/O breakpoints are armed, we need to check if one triggered
15544	* and take appropriate action.
15545	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
15546	*/
15547	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
15548	AssertRCReturn(rc, rc);
15549
15550	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
15551	* execution engines about whether hyper BPs and such are pending. */
15552	uint32_t const uDr7 = pCtx->dr[7];
15553	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
15554	&& X86_DR7_ANY_RW_IO(uDr7)
15555	&& (pCtx->cr4 & X86_CR4_DE))
15556	\|\| DBGFBpIsHwIoArmed(pVM)))
15557	{
15558	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
15559
15560	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
15561	VMMRZCallRing3Disable(pVCpu);
15562	HM_DISABLE_PREEMPT(pVCpu);
15563
15564	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
15565
15566	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
15567	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
15568	{
15569	/* Raise #DB. */
15570	if (fIsGuestDbgActive)
15571	ASMSetDR6(pCtx->dr[6]);
15572	if (pCtx->dr[7] != uDr7)
15573	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
15574
15575	hmR0VmxSetPendingXcptDB(pVCpu);
15576	}
15577	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
15578	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
15579	else if ( rcStrict2 != VINF_SUCCESS
15580	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
15581	rcStrict = rcStrict2;
15582	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
15583
15584	HM_RESTORE_PREEMPT();
15585	VMMRZCallRing3Enable(pVCpu);
15586	}
15587	}
15588
15589	#ifdef VBOX_STRICT
15590	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
15591	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
15592	Assert(!fIOWrite);
15593	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
15594	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
15595	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
15596	Assert(fIOWrite);
15597	else
15598	{
15599	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
15600	* statuses, that the VMM device and some others may return. See
15601	* IOM_SUCCESS() for guidance. */
15602	AssertMsg( RT_FAILURE(rcStrict)
15603	\|\| rcStrict == VINF_SUCCESS
15604	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
15605	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
15606	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
15607	\|\| rcStrict == VINF_EM_RAW_TO_R3
15608	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15609	# endif
15610	}
15611	#endif
15612	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
15613	}
15614	else
15615	{
15616	/*
15617	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
15618	*/
15619	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15620	AssertRCReturn(rc2, rc2);
15621	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
15622	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
15623	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
15624	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15625	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
15626	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
15627
15628	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
15629	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15630
15631	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
15632	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15633	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
15634	}
15635	return rcStrict;
15636	}
15637
15638
15639	/**
15640	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
15641	* VM-exit.
15642	*/
15643	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15644	{
15645	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15646
15647	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
15648	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15649	AssertRCReturn(rc, rc);
15650	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
15651	{
15652	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15653	AssertRCReturn(rc, rc);
15654	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
15655	{
15656	uint32_t uErrCode;
15657	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
15658	{
15659	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15660	AssertRCReturn(rc, rc);
15661	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
15662	}
15663	else
15664	uErrCode = 0;
15665
15666	RTGCUINTPTR GCPtrFaultAddress;
15667	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(pVmxTransient->uIdtVectoringInfo))
15668	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
15669	else
15670	GCPtrFaultAddress = 0;
15671
15672	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15673	AssertRCReturn(rc, rc);
15674
15675	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
15676	pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
15677
15678	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo),
15679	VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo)));
15680	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
15681	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15682	}
15683	}
15684
15685	/* Fall back to the interpreter to emulate the task-switch. */
15686	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
15687	return VERR_EM_INTERPRETER;
15688	}
15689
15690
15691	/**
15692	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
15693	*/
15694	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15695	{
15696	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15697
15698	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15699	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
15700	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
15701	AssertRCReturn(rc, rc);
15702	return VINF_EM_DBG_STEPPED;
15703	}
15704
15705
15706	/**
15707	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
15708	*/
15709	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15710	{
15711	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15712	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
15713
15714	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15715	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15716	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15717	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15718	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15719	AssertRCReturn(rc, rc);
15720
15721	/*
15722	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
15723	*/
15724	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15725	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15726	{
15727	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
15728	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
15729	{
15730	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
15731	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15732	}
15733	}
15734	else
15735	{
15736	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
15737	return rcStrict;
15738	}
15739
15740	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
15741	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15742	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15743	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15744	AssertRCReturn(rc, rc);
15745
15746	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
15747	uint32_t const uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
15748	switch (uAccessType)
15749	{
15750	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
15751	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
15752	{
15753	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
15754	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
15755	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
15756
15757	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
15758	GCPhys &= PAGE_BASE_GC_MASK;
15759	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
15760	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
15761	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
15762
15763	PVM pVM = pVCpu->CTX_SUFF(pVM);
15764	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15765	rcStrict = IOMMMIOPhysHandler(pVM, pVCpu,
15766	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
15767	CPUMCTX2CORE(pCtx), GCPhys);
15768	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15769	if ( rcStrict == VINF_SUCCESS
15770	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
15771	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
15772	{
15773	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
15774	\| HM_CHANGED_GUEST_APIC_TPR);
15775	rcStrict = VINF_SUCCESS;
15776	}
15777	break;
15778	}
15779
15780	default:
15781	{
15782	Log4Func(("uAccessType=%#x\n", uAccessType));
15783	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
15784	break;
15785	}
15786	}
15787
15788	if (rcStrict != VINF_SUCCESS)
15789	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
15790	return rcStrict;
15791	}
15792
15793
15794	/**
15795	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
15796	* VM-exit.
15797	*/
15798	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15799	{
15800	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15801	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15802
15803	/* We might get this VM-exit if the nested-guest is not intercepting MOV DRx accesses. */
15804	if (!pVmxTransient->fIsNestedGuest)
15805	{
15806	/* We should -not- get this VM-exit if the guest's debug registers were active. */
15807	if (pVmxTransient->fWasGuestDebugStateActive)
15808	{
15809	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
15810	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
15811	}
15812
15813	if ( !pVCpu->hm.s.fSingleInstruction
15814	&& !pVmxTransient->fWasHyperDebugStateActive)
15815	{
15816	Assert(!DBGFIsStepping(pVCpu));
15817	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
15818
15819	/* Don't intercept MOV DRx any more. */
15820	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
15821	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
15822	AssertRCReturn(rc, rc);
15823
15824	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
15825	VMMRZCallRing3Disable(pVCpu);
15826	HM_DISABLE_PREEMPT(pVCpu);
15827
15828	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
15829	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
15830	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
15831
15832	HM_RESTORE_PREEMPT();
15833	VMMRZCallRing3Enable(pVCpu);
15834
15835	#ifdef VBOX_WITH_STATISTICS
15836	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15837	AssertRCReturn(rc, rc);
15838	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
15839	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
15840	else
15841	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
15842	#endif
15843	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
15844	return VINF_SUCCESS;
15845	}
15846	}
15847
15848	/*
15849	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
15850	* The EFER MSR is always up-to-date.
15851	* Update the segment registers and DR7 from the CPU.
15852	*/
15853	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15854	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15855	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
15856	AssertRCReturn(rc, rc);
15857	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
15858
15859	PVM pVM = pVCpu->CTX_SUFF(pVM);
15860	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
15861	{
15862	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
15863	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
15864	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
15865	if (RT_SUCCESS(rc))
15866	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
15867	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
15868	}
15869	else
15870	{
15871	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
15872	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
15873	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
15874	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
15875	}
15876
15877	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
15878	if (RT_SUCCESS(rc))
15879	{
15880	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15881	AssertRCReturn(rc2, rc2);
15882	return VINF_SUCCESS;
15883	}
15884	return rc;
15885	}
15886
15887
15888	/**
15889	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
15890	* Conditional VM-exit.
15891	*/
15892	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15893	{
15894	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15895	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
15896
15897	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15898	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15899	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15900	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15901	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15902	AssertRCReturn(rc, rc);
15903
15904	/*
15905	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
15906	*/
15907	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15908	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15909	{
15910	/*
15911	* In the unlikely case where delivering an event causes an EPT misconfig (MMIO), go back to
15912	* instruction emulation to inject the original event. Otherwise, injecting the original event
15913	* using hardware-assisted VMX would would trigger the same EPT misconfig VM-exit again.
15914	*/
15915	if (!pVCpu->hm.s.Event.fPending)
15916	{ /* likely */ }
15917	else
15918	{
15919	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
15920	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15921	/** @todo NSTVMX: Think about how this should be handled. */
15922	if (pVmxTransient->fIsNestedGuest)
15923	return VERR_VMX_IPE_3;
15924	#endif
15925	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15926	}
15927	}
15928	else
15929	{
15930	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
15931	return rcStrict;
15932	}
15933
15934	/*
15935	* Get sufficent state and update the exit history entry.
15936	*/
15937	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15938	rc = hmR0VmxReadGuestPhysicalAddrVmcs(pVCpu, pVmxTransient);
15939	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15940	AssertRCReturn(rc, rc);
15941
15942	RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr;
15943	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
15944	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
15945	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
15946	if (!pExitRec)
15947	{
15948	/*
15949	* If we succeed, resume guest execution.
15950	* If we fail in interpreting the instruction because we couldn't get the guest physical address
15951	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
15952	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
15953	* weird case. See @bugref{6043}.
15954	*/
15955	PVM pVM = pVCpu->CTX_SUFF(pVM);
15956	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15957	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
15958	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
15959	if ( rcStrict == VINF_SUCCESS
15960	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
15961	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
15962	{
15963	/* Successfully handled MMIO operation. */
15964	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
15965	\| HM_CHANGED_GUEST_APIC_TPR);
15966	rcStrict = VINF_SUCCESS;
15967	}
15968	}
15969	else
15970	{
15971	/*
15972	* Frequent exit or something needing probing. Call EMHistoryExec.
15973	*/
15974	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
15975	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
15976
15977	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
15978	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15979
15980	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
15981	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15982	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
15983	}
15984	return rcStrict;
15985	}
15986
15987
15988	/**
15989	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
15990	* VM-exit.
15991	*/
15992	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15993	{
15994	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15995	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
15996
15997	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15998	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15999	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
16000	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16001	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16002	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16003	AssertRCReturn(rc, rc);
16004
16005	/*
16006	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
16007	*/
16008	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
16009	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16010	{
16011	/*
16012	* If delivery of an event causes an EPT violation (true nested #PF and not MMIO),
16013	* we shall resolve the nested #PF and re-inject the original event.
16014	*/
16015	if (pVCpu->hm.s.Event.fPending)
16016	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflectNPF);
16017	}
16018	else
16019	{
16020	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
16021	return rcStrict;
16022	}
16023
16024	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16025	rc = hmR0VmxReadGuestPhysicalAddrVmcs(pVCpu, pVmxTransient);
16026	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
16027	AssertRCReturn(rc, rc);
16028
16029	RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr;
16030	uint64_t const uExitQual = pVmxTransient->uExitQual;
16031	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", uExitQual));
16032
16033	RTGCUINT uErrorCode = 0;
16034	if (uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
16035	uErrorCode \|= X86_TRAP_PF_ID;
16036	if (uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
16037	uErrorCode \|= X86_TRAP_PF_RW;
16038	if (uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
16039	uErrorCode \|= X86_TRAP_PF_P;
16040
16041	PVM pVM = pVCpu->CTX_SUFF(pVM);
16042	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16043	Log4Func(("at %#RX64 (%#RX64 errcode=%#x) cs:rip=%#04x:%#RX64\n", GCPhys, uExitQual, uErrorCode, pCtx->cs.Sel, pCtx->rip));
16044
16045	/*
16046	* Handle the pagefault trap for the nested shadow table.
16047	*/
16048	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
16049	rcStrict = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
16050	TRPMResetTrap(pVCpu);
16051
16052	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
16053	if ( rcStrict == VINF_SUCCESS
16054	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
16055	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
16056	{
16057	/* Successfully synced our nested page tables. */
16058	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
16059	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
16060	return VINF_SUCCESS;
16061	}
16062
16063	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
16064	return rcStrict;
16065	}
16066
16067
16068	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
16069	/**
16070	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
16071	*/
16072	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16073	{
16074	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16075
16076	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16077	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16078	\| CPUMCTX_EXTRN_HWVIRT
16079	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16080	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16081	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16082	AssertRCReturn(rc, rc);
16083
16084	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16085
16086	VMXVEXITINFO ExitInfo;
16087	RT_ZERO(ExitInfo);
16088	ExitInfo.uReason = pVmxTransient->uExitReason;
16089	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16090	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16091	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16092	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16093
16094	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
16095	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16096	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16097	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16098	{
16099	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16100	rcStrict = VINF_SUCCESS;
16101	}
16102	return rcStrict;
16103	}
16104
16105
16106	/**
16107	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
16108	*/
16109	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16110	{
16111	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16112
16113	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
16114	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
16115	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16116	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16117	AssertRCReturn(rc, rc);
16118
16119	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16120
16121	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
16122	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMLAUNCH);
16123	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
16124	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16125	{
16126	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16127	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
16128	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
16129	}
16130	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
16131	return rcStrict;
16132	}
16133
16134
16135	/**
16136	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
16137	*/
16138	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16139	{
16140	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16141
16142	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16143	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16144	\| CPUMCTX_EXTRN_HWVIRT
16145	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16146	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16147	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16148	AssertRCReturn(rc, rc);
16149
16150	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16151
16152	VMXVEXITINFO ExitInfo;
16153	RT_ZERO(ExitInfo);
16154	ExitInfo.uReason = pVmxTransient->uExitReason;
16155	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16156	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16157	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16158	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16159
16160	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
16161	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16162	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16163	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16164	{
16165	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16166	rcStrict = VINF_SUCCESS;
16167	}
16168	return rcStrict;
16169	}
16170
16171
16172	/**
16173	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
16174	*/
16175	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16176	{
16177	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16178
16179	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16180	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16181	\| CPUMCTX_EXTRN_HWVIRT
16182	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16183	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16184	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16185	AssertRCReturn(rc, rc);
16186
16187	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16188
16189	VMXVEXITINFO ExitInfo;
16190	RT_ZERO(ExitInfo);
16191	ExitInfo.uReason = pVmxTransient->uExitReason;
16192	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16193	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16194	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16195	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
16196
16197	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
16198	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16199	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16200	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16201	{
16202	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16203	rcStrict = VINF_SUCCESS;
16204	}
16205	return rcStrict;
16206	}
16207
16208
16209	/**
16210	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Conditional VM-exit.
16211	*/
16212	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16213	{
16214	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16215
16216	/*
16217	* Strictly speaking we should not get VMREAD VM-exits for shadow VMCS fields and
16218	* thus might not need to import the shadow VMCS state, it's safer just in case
16219	* code elsewhere dares look at unsynced VMCS fields.
16220	*/
16221	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16222	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16223	\| CPUMCTX_EXTRN_HWVIRT
16224	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16225	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16226	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16227	AssertRCReturn(rc, rc);
16228
16229	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16230
16231	VMXVEXITINFO ExitInfo;
16232	RT_ZERO(ExitInfo);
16233	ExitInfo.uReason = pVmxTransient->uExitReason;
16234	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16235	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16236	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16237	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16238	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
16239
16240	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
16241	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16242	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16243	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16244	{
16245	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16246	rcStrict = VINF_SUCCESS;
16247	}
16248	return rcStrict;
16249	}
16250
16251
16252	/**
16253	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
16254	*/
16255	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16256	{
16257	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16258
16259	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
16260	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
16261	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16262	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16263	AssertRCReturn(rc, rc);
16264
16265	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16266
16267	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
16268	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMRESUME);
16269	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
16270	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16271	{
16272	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16273	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
16274	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
16275	}
16276	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
16277	return rcStrict;
16278	}
16279
16280
16281	/**
16282	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Conditional VM-exit.
16283	*/
16284	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16285	{
16286	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16287
16288	/*
16289	* Although we should not get VMWRITE VM-exits for shadow VMCS fields, since our HM hook
16290	* gets invoked when IEM's VMWRITE instruction emulation modifies the current VMCS and it
16291	* flags re-loading the entire shadow VMCS, we should save the entire shadow VMCS here.
16292	*/
16293	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16294	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16295	\| CPUMCTX_EXTRN_HWVIRT
16296	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16297	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16298	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16299	AssertRCReturn(rc, rc);
16300
16301	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16302
16303	VMXVEXITINFO ExitInfo;
16304	RT_ZERO(ExitInfo);
16305	ExitInfo.uReason = pVmxTransient->uExitReason;
16306	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16307	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16308	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16309	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16310	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16311
16312	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
16313	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16314	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16315	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16316	{
16317	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16318	rcStrict = VINF_SUCCESS;
16319	}
16320	return rcStrict;
16321	}
16322
16323
16324	/**
16325	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
16326	*/
16327	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16328	{
16329	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16330
16331	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16332	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
16333	\| CPUMCTX_EXTRN_HWVIRT
16334	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
16335	AssertRCReturn(rc, rc);
16336
16337	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16338
16339	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
16340	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16341	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
16342	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16343	{
16344	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16345	rcStrict = VINF_SUCCESS;
16346	}
16347	return rcStrict;
16348	}
16349
16350
16351	/**
16352	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
16353	*/
16354	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16355	{
16356	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16357
16358	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16359	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16360	\| CPUMCTX_EXTRN_HWVIRT
16361	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16362	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16363	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16364	AssertRCReturn(rc, rc);
16365
16366	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16367
16368	VMXVEXITINFO ExitInfo;
16369	RT_ZERO(ExitInfo);
16370	ExitInfo.uReason = pVmxTransient->uExitReason;
16371	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16372	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16373	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16374	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16375
16376	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
16377	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16378	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16379	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16380	{
16381	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16382	rcStrict = VINF_SUCCESS;
16383	}
16384	return rcStrict;
16385	}
16386
16387
16388	/**
16389	* VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
16390	*/
16391	HMVMX_EXIT_DECL hmR0VmxExitInvvpid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16392	{
16393	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16394
16395	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16396	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16397	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16398	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16399	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16400	AssertRCReturn(rc, rc);
16401
16402	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16403
16404	VMXVEXITINFO ExitInfo;
16405	RT_ZERO(ExitInfo);
16406	ExitInfo.uReason = pVmxTransient->uExitReason;
16407	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16408	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16409	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16410	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16411
16412	VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
16413	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16414	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16415	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16416	{
16417	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16418	rcStrict = VINF_SUCCESS;
16419	}
16420	return rcStrict;
16421	}
16422	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
16423	/** @} */
16424
16425
16426	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
16427	/** @name Nested-guest VM-exit handlers.
16428	* @{
16429	*/
16430	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16431	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- Nested-guest VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16432	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16433
16434	/**
16435	* Nested-guest VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
16436	* Conditional VM-exit.
16437	*/
16438	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmiNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16439	{
16440	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16441
16442	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
16443	AssertRCReturn(rc, rc);
16444
16445	uint64_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
16446	uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
16447	Assert(VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo));
16448
16449	switch (uExitIntType)
16450	{
16451	/*
16452	* Physical NMIs:
16453	* We shouldn't direct host physical NMIs to the nested-guest. Dispatch it to the host.
16454	*/
16455	case VMX_EXIT_INT_INFO_TYPE_NMI:
16456	return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
16457
16458	/*
16459	* Hardware exceptions,
16460	* Software exceptions,
16461	* Privileged software exceptions:
16462	* Figure out if the exception must be delivered to the guest or the nested-guest.
16463	*/
16464	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
16465	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
16466	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
16467	{
16468	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
16469	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16470	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16471	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16472	AssertRCReturn(rc, rc);
16473
16474	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16475	bool const fIntercept = CPUMIsGuestVmxXcptInterceptSet(pVCpu, pCtx, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo),
16476	pVmxTransient->uExitIntErrorCode);
16477	if (fIntercept)
16478	{
16479	/* Exit qualification is required for debug and page-fault exceptions. */
16480	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16481	AssertRCReturn(rc, rc);
16482
16483	/*
16484	* For VM-exits due to software exceptions (those generated by INT3 or INTO) and privileged
16485	* software exceptions (those generated by INT1/ICEBP) we need to supply the VM-exit instruction
16486	* length. However, if delivery of a software interrupt, software exception or privileged
16487	* software exception causes a VM-exit, that too provides the VM-exit instruction length.
16488	*/
16489	VMXVEXITINFO ExitInfo;
16490	RT_ZERO(ExitInfo);
16491	ExitInfo.uReason = pVmxTransient->uExitReason;
16492	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16493	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16494
16495	VMXVEXITEVENTINFO ExitEventInfo;
16496	RT_ZERO(ExitEventInfo);
16497	ExitEventInfo.uExitIntInfo = pVmxTransient->uExitIntInfo;
16498	ExitEventInfo.uExitIntErrCode = pVmxTransient->uExitIntErrorCode;
16499	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16500	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16501
16502	#ifdef DEBUG_ramshankar
16503	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
16504	Log4Func(("cs:rip=%#04x:%#RX64 %s err_code=%#x exit_qual=%#RX64\n", pCtx->cs.Sel, pCtx->rip,
16505	VMX_EXIT_INT_INFO_IS_XCPT_PF(pVmxTransient->uExitIntInfo) ? "#PF" : "Unk",
16506	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual));
16507	Log4Func(("idt_info=%#RX64 (%s) idt_errcode=%#RX32\n", pVmxTransient->uIdtVectoringInfo,
16508	VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo) ? "Valid" : "Invalid",
16509	pVmxTransient->uIdtVectoringErrorCode));
16510	#endif
16511	return IEMExecVmxVmexitXcpt(pVCpu, &ExitInfo, &ExitEventInfo);
16512	}
16513
16514	/* Nested paging is currently a requirement, otherwise we would need to handle shadow #PFs in hmR0VmxExitXcptPF. */
16515	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
16516	return hmR0VmxExitXcpt(pVCpu, pVmxTransient);
16517	}
16518
16519	/*
16520	* Software interrupts:
16521	* VM-exits cannot be caused by software interrupts.
16522	*
16523	* External interrupts:
16524	* This should only happen when "acknowledge external interrupts on VM-exit"
16525	* control is set. However, we never set this when executing a guest or
16526	* nested-guest. For nested-guests it is emulated while injecting interrupts into
16527	* the guest.
16528	*/
16529	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
16530	case VMX_EXIT_INT_INFO_TYPE_EXT_INT:
16531	default:
16532	{
16533	pVCpu->hm.s.u32HMError = pVmxTransient->uExitIntInfo;
16534	return VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
16535	}
16536	}
16537	}
16538
16539
16540	/**
16541	* Nested-guest VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT).
16542	* Unconditional VM-exit.
16543	*/
16544	HMVMX_EXIT_DECL hmR0VmxExitTripleFaultNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16545	{
16546	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16547	return IEMExecVmxVmexitTripleFault(pVCpu);
16548	}
16549
16550
16551	/**
16552	* Nested-guest VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
16553	*/
16554	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindowNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16555	{
16556	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16557
16558	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INT_WINDOW_EXIT))
16559	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16560	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
16561	}
16562
16563
16564	/**
16565	* Nested-guest VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
16566	*/
16567	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindowNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16568	{
16569	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16570
16571	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_NMI_WINDOW_EXIT))
16572	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16573	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
16574	}
16575
16576
16577	/**
16578	* Nested-guest VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH).
16579	* Unconditional VM-exit.
16580	*/
16581	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitchNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16582	{
16583	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16584
16585	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16586	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16587	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16588	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16589	AssertRCReturn(rc, rc);
16590
16591	VMXVEXITINFO ExitInfo;
16592	RT_ZERO(ExitInfo);
16593	ExitInfo.uReason = pVmxTransient->uExitReason;
16594	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16595	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16596
16597	VMXVEXITEVENTINFO ExitEventInfo;
16598	RT_ZERO(ExitEventInfo);
16599	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16600	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16601	return IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
16602	}
16603
16604
16605	/**
16606	* Nested-guest VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
16607	*/
16608	HMVMX_EXIT_DECL hmR0VmxExitHltNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16609	{
16610	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16611
16612	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
16613	{
16614	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16615	AssertRCReturn(rc, rc);
16616	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16617	}
16618	return hmR0VmxExitHlt(pVCpu, pVmxTransient);
16619	}
16620
16621
16622	/**
16623	* Nested-guest VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
16624	*/
16625	HMVMX_EXIT_DECL hmR0VmxExitInvlpgNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16626	{
16627	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16628
16629	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
16630	{
16631	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16632	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16633	AssertRCReturn(rc, rc);
16634
16635	VMXVEXITINFO ExitInfo;
16636	RT_ZERO(ExitInfo);
16637	ExitInfo.uReason = pVmxTransient->uExitReason;
16638	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16639	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16640	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16641	}
16642	return hmR0VmxExitInvlpg(pVCpu, pVmxTransient);
16643	}
16644
16645
16646	/**
16647	* Nested-guest VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
16648	*/
16649	HMVMX_EXIT_DECL hmR0VmxExitRdpmcNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16650	{
16651	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16652
16653	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDPMC_EXIT))
16654	{
16655	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16656	AssertRCReturn(rc, rc);
16657	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16658	}
16659	return hmR0VmxExitRdpmc(pVCpu, pVmxTransient);
16660	}
16661
16662
16663	/**
16664	* Nested-guest VM-exit handler for VMREAD (VMX_EXIT_VMREAD) and VMWRITE
16665	* (VMX_EXIT_VMWRITE). Conditional VM-exit.
16666	*/
16667	HMVMX_EXIT_DECL hmR0VmxExitVmreadVmwriteNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16668	{
16669	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16670
16671	Assert( pVmxTransient->uExitReason == VMX_EXIT_VMREAD
16672	\|\| pVmxTransient->uExitReason == VMX_EXIT_VMWRITE);
16673
16674	int rc = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16675	AssertRCReturn(rc, rc);
16676
16677	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
16678	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
16679	uint64_t u64VmcsField = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
16680
16681	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
16682	if (!CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
16683	u64VmcsField &= UINT64_C(0xffffffff);
16684
16685	if (CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, pVmxTransient->uExitReason, u64VmcsField))
16686	{
16687	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16688	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16689	AssertRCReturn(rc, rc);
16690
16691	VMXVEXITINFO ExitInfo;
16692	RT_ZERO(ExitInfo);
16693	ExitInfo.uReason = pVmxTransient->uExitReason;
16694	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16695	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16696	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16697	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16698	}
16699
16700	if (pVmxTransient->uExitReason == VMX_EXIT_VMREAD)
16701	return hmR0VmxExitVmread(pVCpu, pVmxTransient);
16702	return hmR0VmxExitVmwrite(pVCpu, pVmxTransient);
16703	}
16704
16705
16706	/**
16707	* Nested-guest VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
16708	*/
16709	HMVMX_EXIT_DECL hmR0VmxExitRdtscNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16710	{
16711	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16712
16713	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
16714	{
16715	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16716	AssertRCReturn(rc, rc);
16717	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16718	}
16719
16720	return hmR0VmxExitRdtsc(pVCpu, pVmxTransient);
16721	}
16722
16723
16724	/**
16725	* Nested-guest VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX).
16726	* Conditional VM-exit.
16727	*/
16728	HMVMX_EXIT_DECL hmR0VmxExitMovCRxNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16729	{
16730	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16731
16732	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16733	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16734	AssertRCReturn(rc, rc);
16735
16736	VBOXSTRICTRC rcStrict;
16737	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual);
16738	switch (uAccessType)
16739	{
16740	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
16741	{
16742	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
16743	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
16744	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
16745	uint64_t const uNewCrX = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
16746
16747	bool fIntercept;
16748	switch (iCrReg)
16749	{
16750	case 0:
16751	case 4:
16752	fIntercept = CPUMIsGuestVmxMovToCr0Cr4InterceptSet(pVCpu, &pVCpu->cpum.GstCtx, iCrReg, uNewCrX);
16753	break;
16754
16755	case 3:
16756	fIntercept = CPUMIsGuestVmxMovToCr3InterceptSet(pVCpu, uNewCrX);
16757	break;
16758
16759	case 8:
16760	fIntercept = CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_CR8_LOAD_EXIT);
16761	break;
16762
16763	default:
16764	fIntercept = false;
16765	break;
16766	}
16767	if (fIntercept)
16768	{
16769	VMXVEXITINFO ExitInfo;
16770	RT_ZERO(ExitInfo);
16771	ExitInfo.uReason = pVmxTransient->uExitReason;
16772	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16773	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16774	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16775	}
16776	else
16777	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
16778	break;
16779	}
16780
16781	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
16782	{
16783	/*
16784	* CR0/CR4 reads do not cause VM-exits, the read-shadow is used (subject to masking).
16785	* CR2 reads do not cause a VM-exit.
16786	* CR3 reads cause a VM-exit depending on the "CR3 store exiting" control.
16787	* CR8 reads cause a VM-exit depending on the "CR8 store exiting" control.
16788	*/
16789	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
16790	if ( iCrReg == 3
16791	\|\| iCrReg == 8)
16792	{
16793	static const uint32_t s_auCrXReadIntercepts[] = { 0, 0, 0, VMX_PROC_CTLS_CR3_STORE_EXIT, 0,
16794	0, 0, 0, VMX_PROC_CTLS_CR8_STORE_EXIT };
16795	uint32_t const uIntercept = s_auCrXReadIntercepts[iCrReg];
16796	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uIntercept))
16797	{
16798	VMXVEXITINFO ExitInfo;
16799	RT_ZERO(ExitInfo);
16800	ExitInfo.uReason = pVmxTransient->uExitReason;
16801	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16802	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16803	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16804	}
16805	else
16806	{
16807	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
16808	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
16809	}
16810	}
16811	else
16812	{
16813	AssertMsgFailed(("MOV from CR%d VM-exit must not happen\n", iCrReg));
16814	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, iCrReg);
16815	}
16816	break;
16817	}
16818
16819	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
16820	{
16821	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
16822	Assert(pVmcsNstGst);
16823	uint64_t const uGstHostMask = pVmcsNstGst->u64Cr0Mask.u;
16824	uint64_t const uReadShadow = pVmcsNstGst->u64Cr0ReadShadow.u;
16825	if ( (uGstHostMask & X86_CR0_TS)
16826	&& (uReadShadow & X86_CR0_TS))
16827	{
16828	VMXVEXITINFO ExitInfo;
16829	RT_ZERO(ExitInfo);
16830	ExitInfo.uReason = pVmxTransient->uExitReason;
16831	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16832	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16833	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16834	}
16835	else
16836	rcStrict = hmR0VmxExitClts(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr);
16837	break;
16838	}
16839
16840	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
16841	{
16842	RTGCPTR GCPtrEffDst;
16843	uint16_t const uNewMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(pVmxTransient->uExitQual);
16844	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(pVmxTransient->uExitQual);
16845	if (fMemOperand)
16846	{
16847	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
16848	AssertRCReturn(rc, rc);
16849	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
16850	}
16851	else
16852	GCPtrEffDst = NIL_RTGCPTR;
16853
16854	if (CPUMIsGuestVmxLmswInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, uNewMsw))
16855	{
16856	VMXVEXITINFO ExitInfo;
16857	RT_ZERO(ExitInfo);
16858	ExitInfo.uReason = pVmxTransient->uExitReason;
16859	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16860	ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
16861	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16862	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16863	}
16864	else
16865	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, uNewMsw, GCPtrEffDst);
16866	break;
16867	}
16868
16869	default:
16870	{
16871	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
16872	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
16873	}
16874	}
16875
16876	if (rcStrict == VINF_IEM_RAISED_XCPT)
16877	{
16878	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16879	rcStrict = VINF_SUCCESS;
16880	}
16881	return rcStrict;
16882	}
16883
16884
16885	/**
16886	* Nested-guest VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX).
16887	* Conditional VM-exit.
16888	*/
16889	HMVMX_EXIT_DECL hmR0VmxExitMovDRxNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16890	{
16891	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16892
16893	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MOV_DR_EXIT))
16894	{
16895	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16896	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16897	AssertRCReturn(rc, rc);
16898
16899	VMXVEXITINFO ExitInfo;
16900	RT_ZERO(ExitInfo);
16901	ExitInfo.uReason = pVmxTransient->uExitReason;
16902	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16903	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16904	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16905	}
16906	return hmR0VmxExitMovDRx(pVCpu, pVmxTransient);
16907	}
16908
16909
16910	/**
16911	* Nested-guest VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR).
16912	* Conditional VM-exit.
16913	*/
16914	HMVMX_EXIT_DECL hmR0VmxExitIoInstrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16915	{
16916	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16917
16918	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16919	AssertRCReturn(rc, rc);
16920
16921	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
16922	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
16923	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
16924
16925	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
16926	uint8_t const cbAccess = s_aIOSizes[uIOSize];
16927	if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
16928	{
16929	/*
16930	* IN/OUT instruction:
16931	* - Provides VM-exit instruction length.
16932	*
16933	* INS/OUTS instruction:
16934	* - Provides VM-exit instruction length.
16935	* - Provides Guest-linear address.
16936	* - Optionally provides VM-exit instruction info (depends on CPU feature).
16937	*/
16938	PVM pVM = pVCpu->CTX_SUFF(pVM);
16939	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16940	AssertRCReturn(rc, rc);
16941
16942	/* Make sure we don't use stale/uninitialized VMX-transient info. below. */
16943	pVmxTransient->ExitInstrInfo.u = 0;
16944	pVmxTransient->uGuestLinearAddr = 0;
16945
16946	bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
16947	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
16948	if (fIOString)
16949	{
16950	rc \|= hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
16951	if (fVmxInsOutsInfo)
16952	{
16953	Assert(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
16954	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16955	}
16956	}
16957	AssertRCReturn(rc, rc);
16958
16959	VMXVEXITINFO ExitInfo;
16960	RT_ZERO(ExitInfo);
16961	ExitInfo.uReason = pVmxTransient->uExitReason;
16962	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16963	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16964	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16965	ExitInfo.u64GuestLinearAddr = pVmxTransient->uGuestLinearAddr;
16966	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16967	}
16968	return hmR0VmxExitIoInstr(pVCpu, pVmxTransient);
16969	}
16970
16971
16972	/**
16973	* Nested-guest VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
16974	*/
16975	HMVMX_EXIT_DECL hmR0VmxExitRdmsrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16976	{
16977	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16978
16979	uint32_t fMsrpm;
16980	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
16981	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
16982	else
16983	fMsrpm = VMXMSRPM_EXIT_RD;
16984
16985	if (fMsrpm & VMXMSRPM_EXIT_RD)
16986	{
16987	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16988	AssertRCReturn(rc, rc);
16989	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16990	}
16991	return hmR0VmxExitRdmsr(pVCpu, pVmxTransient);
16992	}
16993
16994
16995	/**
16996	* Nested-guest VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
16997	*/
16998	HMVMX_EXIT_DECL hmR0VmxExitWrmsrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16999	{
17000	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17001
17002	uint32_t fMsrpm;
17003	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
17004	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
17005	else
17006	fMsrpm = VMXMSRPM_EXIT_WR;
17007
17008	if (fMsrpm & VMXMSRPM_EXIT_WR)
17009	{
17010	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17011	AssertRCReturn(rc, rc);
17012	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17013	}
17014	return hmR0VmxExitWrmsr(pVCpu, pVmxTransient);
17015	}
17016
17017
17018	/**
17019	* Nested-guest VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
17020	*/
17021	HMVMX_EXIT_DECL hmR0VmxExitMwaitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17022	{
17023	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17024
17025	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MWAIT_EXIT))
17026	{
17027	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17028	AssertRCReturn(rc, rc);
17029	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17030	}
17031	return hmR0VmxExitMwait(pVCpu, pVmxTransient);
17032	}
17033
17034
17035	/**
17036	* Nested-guest VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional
17037	* VM-exit.
17038	*/
17039	HMVMX_EXIT_DECL hmR0VmxExitMtfNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17040	{
17041	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17042
17043	/** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */
17044	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
17045	}
17046
17047
17048	/**
17049	* Nested-guest VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
17050	*/
17051	HMVMX_EXIT_DECL hmR0VmxExitMonitorNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17052	{
17053	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17054
17055	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MONITOR_EXIT))
17056	{
17057	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17058	AssertRCReturn(rc, rc);
17059	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17060	}
17061	return hmR0VmxExitMonitor(pVCpu, pVmxTransient);
17062	}
17063
17064
17065	/**
17066	* Nested-guest VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
17067	*/
17068	HMVMX_EXIT_DECL hmR0VmxExitPauseNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17069	{
17070	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17071
17072	/** @todo NSTVMX: Think about this more. Does the outer guest need to intercept
17073	* PAUSE when executing a nested-guest? If it does not, we would not need
17074	* to check for the intercepts here. Just call VM-exit... */
17075
17076	/* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */
17077	if ( CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT)
17078	\|\| CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
17079	{
17080	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17081	AssertRCReturn(rc, rc);
17082	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17083	}
17084	return hmR0VmxExitPause(pVCpu, pVmxTransient);
17085	}
17086
17087
17088	/**
17089	* Nested-guest VM-exit handler for when the TPR value is lowered below the
17090	* specified threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
17091	*/
17092	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThresholdNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17093	{
17094	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17095
17096	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_TPR_SHADOW))
17097	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
17098	return hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient);
17099	}
17100
17101
17102	/**
17103	* Nested-guest VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional
17104	* VM-exit.
17105	*/
17106	HMVMX_EXIT_DECL hmR0VmxExitApicAccessNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17107	{
17108	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17109
17110	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17111	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
17112	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
17113	AssertRCReturn(rc, rc);
17114
17115	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS));
17116	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17117	AssertRCReturn(rc, rc);
17118
17119	VMXVEXITINFO ExitInfo;
17120	RT_ZERO(ExitInfo);
17121	ExitInfo.uReason = pVmxTransient->uExitReason;
17122	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17123	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17124
17125	VMXVEXITEVENTINFO ExitEventInfo;
17126	RT_ZERO(ExitEventInfo);
17127	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
17128	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
17129	return IEMExecVmxVmexitApicAccess(pVCpu, &ExitInfo, &ExitEventInfo);
17130	}
17131
17132
17133	/**
17134	* Nested-guest VM-exit handler for APIC write emulation (VMX_EXIT_APIC_WRITE).
17135	* Conditional VM-exit.
17136	*/
17137	HMVMX_EXIT_DECL hmR0VmxExitApicWriteNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17138	{
17139	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17140
17141	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_APIC_REG_VIRT));
17142	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17143	AssertRCReturn(rc, rc);
17144
17145	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
17146	}
17147
17148
17149	/**
17150	* Nested-guest VM-exit handler for virtualized EOI (VMX_EXIT_VIRTUALIZED_EOI).
17151	* Conditional VM-exit.
17152	*/
17153	HMVMX_EXIT_DECL hmR0VmxExitVirtEoiNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17154	{
17155	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17156
17157	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_INT_DELIVERY));
17158	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17159	AssertRCReturn(rc, rc);
17160
17161	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
17162	}
17163
17164
17165	/**
17166	* Nested-guest VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
17167	*/
17168	HMVMX_EXIT_DECL hmR0VmxExitRdtscpNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17169	{
17170	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17171
17172	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
17173	{
17174	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_RDTSCP));
17175	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17176	AssertRCReturn(rc, rc);
17177	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17178	}
17179	return hmR0VmxExitRdtscp(pVCpu, pVmxTransient);
17180	}
17181
17182
17183	/**
17184	* Nested-guest VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
17185	*/
17186	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvdNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17187	{
17188	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17189
17190	if (CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
17191	{
17192	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17193	AssertRCReturn(rc, rc);
17194	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17195	}
17196	return hmR0VmxExitWbinvd(pVCpu, pVmxTransient);
17197	}
17198
17199
17200	/**
17201	* Nested-guest VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
17202	*/
17203	HMVMX_EXIT_DECL hmR0VmxExitInvpcidNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17204	{
17205	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17206
17207	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
17208	{
17209	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_INVPCID));
17210	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17211	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17212	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17213	AssertRCReturn(rc, rc);
17214
17215	VMXVEXITINFO ExitInfo;
17216	RT_ZERO(ExitInfo);
17217	ExitInfo.uReason = pVmxTransient->uExitReason;
17218	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17219	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17220	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17221	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17222	}
17223	return hmR0VmxExitInvpcid(pVCpu, pVmxTransient);
17224	}
17225
17226
17227	/**
17228	* Nested-guest VM-exit handler for invalid-guest state
17229	* (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error VM-exit.
17230	*/
17231	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestStateNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17232	{
17233	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17234
17235	/*
17236	* Currently this should never happen because we fully emulate VMLAUNCH/VMRESUME in IEM.
17237	* So if it does happen, it indicates a bug possibly in the hardware-assisted VMX code.
17238	* Handle it like it's in an invalid guest state of the outer guest.
17239	*
17240	* When the fast path is implemented, this should be changed to cause the corresponding
17241	* nested-guest VM-exit.
17242	*/
17243	return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
17244	}
17245
17246
17247	/**
17248	* Nested-guest VM-exit handler for instructions that cause VM-exits uncondtionally
17249	* and only provide the instruction length.
17250	*
17251	* Unconditional VM-exit.
17252	*/
17253	HMVMX_EXIT_DECL hmR0VmxExitInstrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17254	{
17255	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17256
17257	#ifdef VBOX_STRICT
17258	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
17259	switch (pVmxTransient->uExitReason)
17260	{
17261	case VMX_EXIT_ENCLS:
17262	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_ENCLS_EXIT));
17263	break;
17264
17265	case VMX_EXIT_VMFUNC:
17266	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_VMFUNC));
17267	break;
17268	}
17269	#endif
17270
17271	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17272	AssertRCReturn(rc, rc);
17273	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17274	}
17275
17276
17277	/**
17278	* Nested-guest VM-exit handler for instructions that provide instruction length as
17279	* well as more information.
17280	*
17281	* Unconditional VM-exit.
17282	*/
17283	HMVMX_EXIT_DECL hmR0VmxExitInstrWithInfoNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17284	{
17285	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17286
17287	#ifdef VBOX_STRICT
17288	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
17289	switch (pVmxTransient->uExitReason)
17290	{
17291	case VMX_EXIT_GDTR_IDTR_ACCESS:
17292	case VMX_EXIT_LDTR_TR_ACCESS:
17293	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_DESC_TABLE_EXIT));
17294	break;
17295
17296	case VMX_EXIT_RDRAND:
17297	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDRAND_EXIT));
17298	break;
17299
17300	case VMX_EXIT_RDSEED:
17301	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDSEED_EXIT));
17302	break;
17303
17304	case VMX_EXIT_XSAVES:
17305	case VMX_EXIT_XRSTORS:
17306	/** @todo NSTVMX: Verify XSS-bitmap. */
17307	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_XSAVES_XRSTORS));
17308	break;
17309
17310	case VMX_EXIT_UMWAIT:
17311	case VMX_EXIT_TPAUSE:
17312	Assert(CPUMIsGuestVmxProcCtlsSet(pVCpu, pCtx, VMX_PROC_CTLS_RDTSC_EXIT));
17313	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_USER_WAIT_PAUSE));
17314	break;
17315	}
17316	#endif
17317
17318	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17319	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17320	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17321	AssertRCReturn(rc, rc);
17322
17323	VMXVEXITINFO ExitInfo;
17324	RT_ZERO(ExitInfo);
17325	ExitInfo.uReason = pVmxTransient->uExitReason;
17326	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17327	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17328	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17329	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17330	}
17331
17332	/** @} */
17333	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
17334

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 80090

Download in other formats: