HMVMXR0.cpp@ 80094

Last change on this file since 80094 was 80094, checked in by vboxsync, 6 years ago
VMM/HM: Kicking out 32-bit host support - VMX. bugref:9511
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 714.7 KB

Line
1	/* $Id: HMVMXR0.cpp 80094 2019-08-01 04:25:23Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27	#include <iprt/mem.h>
28	#include <iprt/mp.h>
29
30	#include <VBox/vmm/pdmapi.h>
31	#include <VBox/vmm/dbgf.h>
32	#include <VBox/vmm/iem.h>
33	#include <VBox/vmm/iom.h>
34	#include <VBox/vmm/selm.h>
35	#include <VBox/vmm/tm.h>
36	#include <VBox/vmm/em.h>
37	#include <VBox/vmm/gim.h>
38	#include <VBox/vmm/apic.h>
39	#ifdef VBOX_WITH_REM
40	# include <VBox/vmm/rem.h>
41	#endif
42	#include "HMInternal.h"
43	#include <VBox/vmm/vm.h>
44	#include <VBox/vmm/hmvmxinline.h>
45	#include "HMVMXR0.h"
46	#include "dtrace/VBoxVMM.h"
47
48	#ifdef DEBUG_ramshankar
49	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
50	# define HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
51	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
52	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
53	# define HMVMX_ALWAYS_CLEAN_TRANSIENT
54	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
55	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
56	# define HMVMX_ALWAYS_TRAP_PF
57	# define HMVMX_ALWAYS_FLUSH_TLB
58	# define HMVMX_ALWAYS_SWAP_EFER
59	#endif
60
61
62	/*********************************************************************************************************************************
63	* Defined Constants And Macros *
64	*********************************************************************************************************************************/
65	/** Use the function table. */
66	#define HMVMX_USE_FUNCTION_TABLE
67
68	/** Determine which tagged-TLB flush handler to use. */
69	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
70	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
71	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
72	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
73
74	/**
75	* Flags to skip redundant reads of some common VMCS fields that are not part of
76	* the guest-CPU or VCPU state but are needed while handling VM-exits.
77	*/
78	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
79	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
80	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
81	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
82	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
83	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
84	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
85	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
86	#define HMVMX_READ_GUEST_PHYSICAL_ADDR RT_BIT_32(8)
87
88	/** All the VMCS fields required for processing of exception/NMI VM-exits. */
89	#define HMVMX_READ_XCPT_INFO ( HMVMX_READ_EXIT_INTERRUPTION_INFO \
90	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE \
91	\| HMVMX_READ_EXIT_INSTR_LEN \
92	\| HMVMX_READ_IDT_VECTORING_INFO \
93	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE)
94
95	/** Assert that all the given fields have been read from the VMCS. */
96	#ifdef VBOX_STRICT
97	# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) \
98	do { \
99	uint32_t const fVmcsFieldRead = ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead); \
100	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE(); \
101	Assert((fVmcsFieldRead & (a_fReadFields)) == (a_fReadFields)); \
102	} while (0)
103	#else
104	# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) do { } while (0)
105	#endif
106
107	/**
108	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
109	* guest using hardware-assisted VMX.
110	*
111	* This excludes state like GPRs (other than RSP) which are always are
112	* swapped and restored across the world-switch and also registers like EFER,
113	* MSR which cannot be modified by the guest without causing a VM-exit.
114	*/
115	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
116	\| CPUMCTX_EXTRN_RFLAGS \
117	\| CPUMCTX_EXTRN_RSP \
118	\| CPUMCTX_EXTRN_SREG_MASK \
119	\| CPUMCTX_EXTRN_TABLE_MASK \
120	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
121	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
122	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
123	\| CPUMCTX_EXTRN_TSC_AUX \
124	\| CPUMCTX_EXTRN_OTHER_MSRS \
125	\| CPUMCTX_EXTRN_CR0 \
126	\| CPUMCTX_EXTRN_CR3 \
127	\| CPUMCTX_EXTRN_CR4 \
128	\| CPUMCTX_EXTRN_DR7 \
129	\| CPUMCTX_EXTRN_HWVIRT \
130	\| CPUMCTX_EXTRN_HM_VMX_MASK)
131
132	/**
133	* Exception bitmap mask for real-mode guests (real-on-v86).
134	*
135	* We need to intercept all exceptions manually except:
136	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
137	* due to bugs in Intel CPUs.
138	* - \#PF need not be intercepted even in real-mode if we have nested paging
139	* support.
140	*/
141	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
142	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
143	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
144	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
145	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
146	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
147	\| RT_BIT(X86_XCPT_XF))
148
149	/** Maximum VM-instruction error number. */
150	#define HMVMX_INSTR_ERROR_MAX 28
151
152	/** Profiling macro. */
153	#ifdef HM_PROFILE_EXIT_DISPATCH
154	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
155	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
156	#else
157	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
158	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
159	#endif
160
161	/** Assert that preemption is disabled or covered by thread-context hooks. */
162	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
163	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
164
165	/** Assert that we haven't migrated CPUs when thread-context hooks are not
166	* used. */
167	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
168	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
169	("Illegal migration! Entered on CPU %u Current %u\n", \
170	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
171
172	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
173	* context. */
174	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
175	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
176	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
177
178	/** Log the VM-exit reason with an easily visible marker to identify it in a
179	* potential sea of logging data. */
180	#define HMVMX_LOG_EXIT(a_pVCpu, a_uExitReason) \
181	do { \
182	Log4(("VM-exit: vcpu[%RU32] %85s -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (a_pVCpu)->idCpu, \
183	HMGetVmxExitName(a_uExitReason))); \
184	} while (0) \
185
186
187	/*********************************************************************************************************************************
188	* Structures and Typedefs *
189	*********************************************************************************************************************************/
190	/**
191	* VMX per-VCPU transient state.
192	*
193	* A state structure for holding miscellaneous information across
194	* VMX non-root operation and restored after the transition.
195	*/
196	typedef struct VMXTRANSIENT
197	{
198	/** The host's rflags/eflags. */
199	RTCCUINTREG fEFlags;
200	#if HC_ARCH_BITS == 32
201	uint32_t u32Alignment0;
202	#endif
203	/** The guest's TPR value used for TPR shadowing. */
204	uint8_t u8GuestTpr;
205	/** Alignment. */
206	uint8_t abAlignment0[7];
207
208	/** The basic VM-exit reason. */
209	uint16_t uExitReason;
210	/** Alignment. */
211	uint16_t u16Alignment0;
212	/** The VM-exit interruption error code. */
213	uint32_t uExitIntErrorCode;
214	/** The VM-exit exit code qualification. */
215	uint64_t uExitQual;
216	/** The Guest-linear address. */
217	uint64_t uGuestLinearAddr;
218	/** The Guest-physical address. */
219	uint64_t uGuestPhysicalAddr;
220
221	/** The VM-exit interruption-information field. */
222	uint32_t uExitIntInfo;
223	/** The VM-exit instruction-length field. */
224	uint32_t cbInstr;
225	/** The VM-exit instruction-information field. */
226	VMXEXITINSTRINFO ExitInstrInfo;
227	/** Whether the VM-entry failed or not. */
228	bool fVMEntryFailed;
229	/** Whether we are currently executing a nested-guest. */
230	bool fIsNestedGuest;
231	/** Alignment. */
232	uint8_t abAlignment1[2];
233
234	/** The VM-entry interruption-information field. */
235	uint32_t uEntryIntInfo;
236	/** The VM-entry exception error code field. */
237	uint32_t uEntryXcptErrorCode;
238	/** The VM-entry instruction length field. */
239	uint32_t cbEntryInstr;
240
241	/** IDT-vectoring information field. */
242	uint32_t uIdtVectoringInfo;
243	/** IDT-vectoring error code. */
244	uint32_t uIdtVectoringErrorCode;
245
246	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
247	uint32_t fVmcsFieldsRead;
248
249	/** Whether the guest debug state was active at the time of VM-exit. */
250	bool fWasGuestDebugStateActive;
251	/** Whether the hyper debug state was active at the time of VM-exit. */
252	bool fWasHyperDebugStateActive;
253	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
254	bool fUpdatedTscOffsettingAndPreemptTimer;
255	/** Whether the VM-exit was caused by a page-fault during delivery of a
256	* contributory exception or a page-fault. */
257	bool fVectoringDoublePF;
258	/** Whether the VM-exit was caused by a page-fault during delivery of an
259	* external interrupt or NMI. */
260	bool fVectoringPF;
261	/** Whether the TSC_AUX MSR needs to be removed from the auto-load/store MSR
262	* area after VM-exit. */
263	bool fRemoveTscAuxMsr;
264	bool afAlignment0[2];
265
266	/** The VMCS info. object. */
267	PVMXVMCSINFO pVmcsInfo;
268	} VMXTRANSIENT;
269	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
270	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
271	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
272	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
273	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
274	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
275	/** Pointer to VMX transient state. */
276	typedef VMXTRANSIENT *PVMXTRANSIENT;
277	/** Pointer to a const VMX transient state. */
278	typedef const VMXTRANSIENT *PCVMXTRANSIENT;
279
280	/**
281	* Memory operand read or write access.
282	*/
283	typedef enum VMXMEMACCESS
284	{
285	VMXMEMACCESS_READ = 0,
286	VMXMEMACCESS_WRITE = 1
287	} VMXMEMACCESS;
288
289	/**
290	* VMX VM-exit handler.
291	*
292	* @returns Strict VBox status code (i.e. informational status codes too).
293	* @param pVCpu The cross context virtual CPU structure.
294	* @param pVmxTransient The VMX-transient structure.
295	*/
296	#ifndef HMVMX_USE_FUNCTION_TABLE
297	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
298	#else
299	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
300	/** Pointer to VM-exit handler. */
301	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
302	#endif
303
304	/**
305	* VMX VM-exit handler, non-strict status code.
306	*
307	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
308	*
309	* @returns VBox status code, no informational status code returned.
310	* @param pVCpu The cross context virtual CPU structure.
311	* @param pVmxTransient The VMX-transient structure.
312	*
313	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
314	* use of that status code will be replaced with VINF_EM_SOMETHING
315	* later when switching over to IEM.
316	*/
317	#ifndef HMVMX_USE_FUNCTION_TABLE
318	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
319	#else
320	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
321	#endif
322
323
324	/*********************************************************************************************************************************
325	* Internal Functions *
326	*********************************************************************************************************************************/
327	#ifndef HMVMX_USE_FUNCTION_TABLE
328	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
329	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
330	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
331	#else
332	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
333	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
334	#endif
335	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
336	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
337	#endif
338
339	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
340
341	/** @name VM-exit handler prototypes.
342	* @{
343	*/
344	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
345	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
346	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
347	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
348	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
349	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
350	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
351	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
352	static FNVMXEXITHANDLER hmR0VmxExitHlt;
353	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
354	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
355	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
356	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
357	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
358	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
359	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
360	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
361	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
362	static FNVMXEXITHANDLER hmR0VmxExitVmread;
363	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
364	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
365	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
366	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
367	static FNVMXEXITHANDLER hmR0VmxExitInvvpid;
368	#endif
369	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
370	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
371	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
372	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
373	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
374	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
375	static FNVMXEXITHANDLER hmR0VmxExitMwait;
376	static FNVMXEXITHANDLER hmR0VmxExitMtf;
377	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
378	static FNVMXEXITHANDLER hmR0VmxExitPause;
379	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
380	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
381	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
382	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
383	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
384	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
385	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
386	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
387	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
388	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
389	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
390	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUnexpected;
391	/** @} */
392
393	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
394	/** @name Nested-guest VM-exit handler prototypes.
395	* @{
396	*/
397	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmiNested;
398	static FNVMXEXITHANDLER hmR0VmxExitTripleFaultNested;
399	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindowNested;
400	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindowNested;
401	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitchNested;
402	static FNVMXEXITHANDLER hmR0VmxExitHltNested;
403	static FNVMXEXITHANDLER hmR0VmxExitInvlpgNested;
404	static FNVMXEXITHANDLER hmR0VmxExitRdpmcNested;
405	static FNVMXEXITHANDLER hmR0VmxExitVmreadVmwriteNested;
406	static FNVMXEXITHANDLER hmR0VmxExitRdtscNested;
407	static FNVMXEXITHANDLER hmR0VmxExitMovCRxNested;
408	static FNVMXEXITHANDLER hmR0VmxExitMovDRxNested;
409	static FNVMXEXITHANDLER hmR0VmxExitIoInstrNested;
410	static FNVMXEXITHANDLER hmR0VmxExitRdmsrNested;
411	static FNVMXEXITHANDLER hmR0VmxExitWrmsrNested;
412	static FNVMXEXITHANDLER hmR0VmxExitMwaitNested;
413	static FNVMXEXITHANDLER hmR0VmxExitMtfNested;
414	static FNVMXEXITHANDLER hmR0VmxExitMonitorNested;
415	static FNVMXEXITHANDLER hmR0VmxExitPauseNested;
416	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThresholdNested;
417	static FNVMXEXITHANDLER hmR0VmxExitApicAccessNested;
418	static FNVMXEXITHANDLER hmR0VmxExitApicWriteNested;
419	static FNVMXEXITHANDLER hmR0VmxExitVirtEoiNested;
420	static FNVMXEXITHANDLER hmR0VmxExitRdtscpNested;
421	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvdNested;
422	static FNVMXEXITHANDLER hmR0VmxExitInvpcidNested;
423	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestStateNested;
424	static FNVMXEXITHANDLER hmR0VmxExitInstrNested;
425	static FNVMXEXITHANDLER hmR0VmxExitInstrWithInfoNested;
426	/** @} */
427	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
428
429
430	/*********************************************************************************************************************************
431	* Global Variables *
432	*********************************************************************************************************************************/
433	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
434	/**
435	* Array of all VMCS fields.
436	* Any fields added to the VT-x spec. should be added here.
437	*
438	* Currently only used to derive shadow VMCS fields for hardware-assisted execution
439	* of nested-guests.
440	*/
441	static const uint32_t g_aVmcsFields[] =
442	{
443	/* 16-bit control fields. */
444	VMX_VMCS16_VPID,
445	VMX_VMCS16_POSTED_INT_NOTIFY_VECTOR,
446	VMX_VMCS16_EPTP_INDEX,
447
448	/* 16-bit guest-state fields. */
449	VMX_VMCS16_GUEST_ES_SEL,
450	VMX_VMCS16_GUEST_CS_SEL,
451	VMX_VMCS16_GUEST_SS_SEL,
452	VMX_VMCS16_GUEST_DS_SEL,
453	VMX_VMCS16_GUEST_FS_SEL,
454	VMX_VMCS16_GUEST_GS_SEL,
455	VMX_VMCS16_GUEST_LDTR_SEL,
456	VMX_VMCS16_GUEST_TR_SEL,
457	VMX_VMCS16_GUEST_INTR_STATUS,
458	VMX_VMCS16_GUEST_PML_INDEX,
459
460	/* 16-bits host-state fields. */
461	VMX_VMCS16_HOST_ES_SEL,
462	VMX_VMCS16_HOST_CS_SEL,
463	VMX_VMCS16_HOST_SS_SEL,
464	VMX_VMCS16_HOST_DS_SEL,
465	VMX_VMCS16_HOST_FS_SEL,
466	VMX_VMCS16_HOST_GS_SEL,
467	VMX_VMCS16_HOST_TR_SEL,
468
469	/* 64-bit control fields. */
470	VMX_VMCS64_CTRL_IO_BITMAP_A_FULL,
471	VMX_VMCS64_CTRL_IO_BITMAP_A_HIGH,
472	VMX_VMCS64_CTRL_IO_BITMAP_B_FULL,
473	VMX_VMCS64_CTRL_IO_BITMAP_B_HIGH,
474	VMX_VMCS64_CTRL_MSR_BITMAP_FULL,
475	VMX_VMCS64_CTRL_MSR_BITMAP_HIGH,
476	VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL,
477	VMX_VMCS64_CTRL_EXIT_MSR_STORE_HIGH,
478	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL,
479	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_HIGH,
480	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL,
481	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_HIGH,
482	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL,
483	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_HIGH,
484	VMX_VMCS64_CTRL_EXEC_PML_ADDR_FULL,
485	VMX_VMCS64_CTRL_EXEC_PML_ADDR_HIGH,
486	VMX_VMCS64_CTRL_TSC_OFFSET_FULL,
487	VMX_VMCS64_CTRL_TSC_OFFSET_HIGH,
488	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL,
489	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_HIGH,
490	VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL,
491	VMX_VMCS64_CTRL_APIC_ACCESSADDR_HIGH,
492	VMX_VMCS64_CTRL_POSTED_INTR_DESC_FULL,
493	VMX_VMCS64_CTRL_POSTED_INTR_DESC_HIGH,
494	VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL,
495	VMX_VMCS64_CTRL_VMFUNC_CTRLS_HIGH,
496	VMX_VMCS64_CTRL_EPTP_FULL,
497	VMX_VMCS64_CTRL_EPTP_HIGH,
498	VMX_VMCS64_CTRL_EOI_BITMAP_0_FULL,
499	VMX_VMCS64_CTRL_EOI_BITMAP_0_HIGH,
500	VMX_VMCS64_CTRL_EOI_BITMAP_1_FULL,
501	VMX_VMCS64_CTRL_EOI_BITMAP_1_HIGH,
502	VMX_VMCS64_CTRL_EOI_BITMAP_2_FULL,
503	VMX_VMCS64_CTRL_EOI_BITMAP_2_HIGH,
504	VMX_VMCS64_CTRL_EOI_BITMAP_3_FULL,
505	VMX_VMCS64_CTRL_EOI_BITMAP_3_HIGH,
506	VMX_VMCS64_CTRL_EPTP_LIST_FULL,
507	VMX_VMCS64_CTRL_EPTP_LIST_HIGH,
508	VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL,
509	VMX_VMCS64_CTRL_VMREAD_BITMAP_HIGH,
510	VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL,
511	VMX_VMCS64_CTRL_VMWRITE_BITMAP_HIGH,
512	VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_FULL,
513	VMX_VMCS64_CTRL_VIRTXCPT_INFO_ADDR_HIGH,
514	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_FULL,
515	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_HIGH,
516	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_FULL,
517	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_HIGH,
518	VMX_VMCS64_CTRL_TSC_MULTIPLIER_FULL,
519	VMX_VMCS64_CTRL_TSC_MULTIPLIER_HIGH,
520
521	/* 64-bit read-only data fields. */
522	VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL,
523	VMX_VMCS64_RO_GUEST_PHYS_ADDR_HIGH,
524
525	/* 64-bit guest-state fields. */
526	VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL,
527	VMX_VMCS64_GUEST_VMCS_LINK_PTR_HIGH,
528	VMX_VMCS64_GUEST_DEBUGCTL_FULL,
529	VMX_VMCS64_GUEST_DEBUGCTL_HIGH,
530	VMX_VMCS64_GUEST_PAT_FULL,
531	VMX_VMCS64_GUEST_PAT_HIGH,
532	VMX_VMCS64_GUEST_EFER_FULL,
533	VMX_VMCS64_GUEST_EFER_HIGH,
534	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL,
535	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_HIGH,
536	VMX_VMCS64_GUEST_PDPTE0_FULL,
537	VMX_VMCS64_GUEST_PDPTE0_HIGH,
538	VMX_VMCS64_GUEST_PDPTE1_FULL,
539	VMX_VMCS64_GUEST_PDPTE1_HIGH,
540	VMX_VMCS64_GUEST_PDPTE2_FULL,
541	VMX_VMCS64_GUEST_PDPTE2_HIGH,
542	VMX_VMCS64_GUEST_PDPTE3_FULL,
543	VMX_VMCS64_GUEST_PDPTE3_HIGH,
544	VMX_VMCS64_GUEST_BNDCFGS_FULL,
545	VMX_VMCS64_GUEST_BNDCFGS_HIGH,
546
547	/* 64-bit host-state fields. */
548	VMX_VMCS64_HOST_PAT_FULL,
549	VMX_VMCS64_HOST_PAT_HIGH,
550	VMX_VMCS64_HOST_EFER_FULL,
551	VMX_VMCS64_HOST_EFER_HIGH,
552	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL,
553	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_HIGH,
554
555	/* 32-bit control fields. */
556	VMX_VMCS32_CTRL_PIN_EXEC,
557	VMX_VMCS32_CTRL_PROC_EXEC,
558	VMX_VMCS32_CTRL_EXCEPTION_BITMAP,
559	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK,
560	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH,
561	VMX_VMCS32_CTRL_CR3_TARGET_COUNT,
562	VMX_VMCS32_CTRL_EXIT,
563	VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT,
564	VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT,
565	VMX_VMCS32_CTRL_ENTRY,
566	VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT,
567	VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO,
568	VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE,
569	VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH,
570	VMX_VMCS32_CTRL_TPR_THRESHOLD,
571	VMX_VMCS32_CTRL_PROC_EXEC2,
572	VMX_VMCS32_CTRL_PLE_GAP,
573	VMX_VMCS32_CTRL_PLE_WINDOW,
574
575	/* 32-bits read-only fields. */
576	VMX_VMCS32_RO_VM_INSTR_ERROR,
577	VMX_VMCS32_RO_EXIT_REASON,
578	VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO,
579	VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE,
580	VMX_VMCS32_RO_IDT_VECTORING_INFO,
581	VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE,
582	VMX_VMCS32_RO_EXIT_INSTR_LENGTH,
583	VMX_VMCS32_RO_EXIT_INSTR_INFO,
584
585	/* 32-bit guest-state fields. */
586	VMX_VMCS32_GUEST_ES_LIMIT,
587	VMX_VMCS32_GUEST_CS_LIMIT,
588	VMX_VMCS32_GUEST_SS_LIMIT,
589	VMX_VMCS32_GUEST_DS_LIMIT,
590	VMX_VMCS32_GUEST_FS_LIMIT,
591	VMX_VMCS32_GUEST_GS_LIMIT,
592	VMX_VMCS32_GUEST_LDTR_LIMIT,
593	VMX_VMCS32_GUEST_TR_LIMIT,
594	VMX_VMCS32_GUEST_GDTR_LIMIT,
595	VMX_VMCS32_GUEST_IDTR_LIMIT,
596	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
597	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
598	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
599	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
600	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
601	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS,
602	VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS,
603	VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS,
604	VMX_VMCS32_GUEST_INT_STATE,
605	VMX_VMCS32_GUEST_ACTIVITY_STATE,
606	VMX_VMCS32_GUEST_SMBASE,
607	VMX_VMCS32_GUEST_SYSENTER_CS,
608	VMX_VMCS32_PREEMPT_TIMER_VALUE,
609
610	/* 32-bit host-state fields. */
611	VMX_VMCS32_HOST_SYSENTER_CS,
612
613	/* Natural-width control fields. */
614	VMX_VMCS_CTRL_CR0_MASK,
615	VMX_VMCS_CTRL_CR4_MASK,
616	VMX_VMCS_CTRL_CR0_READ_SHADOW,
617	VMX_VMCS_CTRL_CR4_READ_SHADOW,
618	VMX_VMCS_CTRL_CR3_TARGET_VAL0,
619	VMX_VMCS_CTRL_CR3_TARGET_VAL1,
620	VMX_VMCS_CTRL_CR3_TARGET_VAL2,
621	VMX_VMCS_CTRL_CR3_TARGET_VAL3,
622
623	/* Natural-width read-only data fields. */
624	VMX_VMCS_RO_EXIT_QUALIFICATION,
625	VMX_VMCS_RO_IO_RCX,
626	VMX_VMCS_RO_IO_RSI,
627	VMX_VMCS_RO_IO_RDI,
628	VMX_VMCS_RO_IO_RIP,
629	VMX_VMCS_RO_GUEST_LINEAR_ADDR,
630
631	/* Natural-width guest-state field */
632	VMX_VMCS_GUEST_CR0,
633	VMX_VMCS_GUEST_CR3,
634	VMX_VMCS_GUEST_CR4,
635	VMX_VMCS_GUEST_ES_BASE,
636	VMX_VMCS_GUEST_CS_BASE,
637	VMX_VMCS_GUEST_SS_BASE,
638	VMX_VMCS_GUEST_DS_BASE,
639	VMX_VMCS_GUEST_FS_BASE,
640	VMX_VMCS_GUEST_GS_BASE,
641	VMX_VMCS_GUEST_LDTR_BASE,
642	VMX_VMCS_GUEST_TR_BASE,
643	VMX_VMCS_GUEST_GDTR_BASE,
644	VMX_VMCS_GUEST_IDTR_BASE,
645	VMX_VMCS_GUEST_DR7,
646	VMX_VMCS_GUEST_RSP,
647	VMX_VMCS_GUEST_RIP,
648	VMX_VMCS_GUEST_RFLAGS,
649	VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,
650	VMX_VMCS_GUEST_SYSENTER_ESP,
651	VMX_VMCS_GUEST_SYSENTER_EIP,
652
653	/* Natural-width host-state fields */
654	VMX_VMCS_HOST_CR0,
655	VMX_VMCS_HOST_CR3,
656	VMX_VMCS_HOST_CR4,
657	VMX_VMCS_HOST_FS_BASE,
658	VMX_VMCS_HOST_GS_BASE,
659	VMX_VMCS_HOST_TR_BASE,
660	VMX_VMCS_HOST_GDTR_BASE,
661	VMX_VMCS_HOST_IDTR_BASE,
662	VMX_VMCS_HOST_SYSENTER_ESP,
663	VMX_VMCS_HOST_SYSENTER_EIP,
664	VMX_VMCS_HOST_RSP,
665	VMX_VMCS_HOST_RIP
666	};
667	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
668
669	static const uint32_t g_aVmcsSegBase[] =
670	{
671	VMX_VMCS_GUEST_ES_BASE,
672	VMX_VMCS_GUEST_CS_BASE,
673	VMX_VMCS_GUEST_SS_BASE,
674	VMX_VMCS_GUEST_DS_BASE,
675	VMX_VMCS_GUEST_FS_BASE,
676	VMX_VMCS_GUEST_GS_BASE
677	};
678	static const uint32_t g_aVmcsSegSel[] =
679	{
680	VMX_VMCS16_GUEST_ES_SEL,
681	VMX_VMCS16_GUEST_CS_SEL,
682	VMX_VMCS16_GUEST_SS_SEL,
683	VMX_VMCS16_GUEST_DS_SEL,
684	VMX_VMCS16_GUEST_FS_SEL,
685	VMX_VMCS16_GUEST_GS_SEL
686	};
687	static const uint32_t g_aVmcsSegLimit[] =
688	{
689	VMX_VMCS32_GUEST_ES_LIMIT,
690	VMX_VMCS32_GUEST_CS_LIMIT,
691	VMX_VMCS32_GUEST_SS_LIMIT,
692	VMX_VMCS32_GUEST_DS_LIMIT,
693	VMX_VMCS32_GUEST_FS_LIMIT,
694	VMX_VMCS32_GUEST_GS_LIMIT
695	};
696	static const uint32_t g_aVmcsSegAttr[] =
697	{
698	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
699	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
700	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
701	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
702	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
703	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
704	};
705	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
706	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
707	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
708	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
709
710	#ifdef HMVMX_USE_FUNCTION_TABLE
711	/**
712	* VMX_EXIT dispatch table.
713	*/
714	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
715	{
716	/* 0 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
717	/* 1 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
718	/* 2 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
719	/* 3 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitErrUnexpected,
720	/* 4 VMX_EXIT_SIPI */ hmR0VmxExitErrUnexpected,
721	/* 5 VMX_EXIT_IO_SMI */ hmR0VmxExitErrUnexpected,
722	/* 6 VMX_EXIT_SMI */ hmR0VmxExitErrUnexpected,
723	/* 7 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
724	/* 8 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
725	/* 9 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
726	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
727	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
728	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
729	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
730	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
731	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
732	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
733	/* 17 VMX_EXIT_RSM */ hmR0VmxExitErrUnexpected,
734	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
735	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
736	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
737	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
738	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
739	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
740	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
741	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
742	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
743	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
744	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
745	#else
746	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
747	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
748	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
749	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
750	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
751	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
752	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
753	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
754	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
755	#endif
756	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
757	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
758	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
759	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
760	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
761	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
762	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrUnexpected,
763	/* 35 UNDEFINED */ hmR0VmxExitErrUnexpected,
764	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
765	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
766	/* 38 UNDEFINED */ hmR0VmxExitErrUnexpected,
767	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
768	/* 40 VMX_EXIT_PAUSE */ hmR0VmxExitPause,
769	/* 41 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUnexpected,
770	/* 42 UNDEFINED */ hmR0VmxExitErrUnexpected,
771	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
772	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
773	/* 45 VMX_EXIT_VIRTUALIZED_EOI */ hmR0VmxExitErrUnexpected,
774	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitErrUnexpected,
775	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitErrUnexpected,
776	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
777	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
778	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
779	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
780	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
781	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
782	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInvvpid,
783	#else
784	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
785	#endif
786	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
787	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
788	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUnexpected,
789	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitErrUnexpected,
790	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
791	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitErrUnexpected,
792	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUnexpected,
793	/* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUnexpected,
794	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUnexpected,
795	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitErrUnexpected,
796	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitErrUnexpected,
797	/* 65 UNDEFINED */ hmR0VmxExitErrUnexpected,
798	/* 66 VMX_EXIT_SPP_EVENT */ hmR0VmxExitErrUnexpected,
799	/* 67 VMX_EXIT_UMWAIT */ hmR0VmxExitErrUnexpected,
800	/* 68 VMX_EXIT_TPAUSE */ hmR0VmxExitErrUnexpected,
801	};
802	#endif /* HMVMX_USE_FUNCTION_TABLE */
803
804	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
805	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
806	{
807	/* 0 */ "(Not Used)",
808	/* 1 */ "VMCALL executed in VMX root operation.",
809	/* 2 */ "VMCLEAR with invalid physical address.",
810	/* 3 */ "VMCLEAR with VMXON pointer.",
811	/* 4 */ "VMLAUNCH with non-clear VMCS.",
812	/* 5 */ "VMRESUME with non-launched VMCS.",
813	/* 6 */ "VMRESUME after VMXOFF",
814	/* 7 */ "VM-entry with invalid control fields.",
815	/* 8 */ "VM-entry with invalid host state fields.",
816	/* 9 */ "VMPTRLD with invalid physical address.",
817	/* 10 */ "VMPTRLD with VMXON pointer.",
818	/* 11 */ "VMPTRLD with incorrect revision identifier.",
819	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
820	/* 13 */ "VMWRITE to read-only VMCS component.",
821	/* 14 */ "(Not Used)",
822	/* 15 */ "VMXON executed in VMX root operation.",
823	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
824	/* 17 */ "VM-entry with non-launched executing VMCS.",
825	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
826	/* 19 */ "VMCALL with non-clear VMCS.",
827	/* 20 */ "VMCALL with invalid VM-exit control fields.",
828	/* 21 */ "(Not Used)",
829	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
830	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
831	/* 24 */ "VMCALL with invalid SMM-monitor features.",
832	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
833	/* 26 */ "VM-entry with events blocked by MOV SS.",
834	/* 27 */ "(Not Used)",
835	/* 28 */ "Invalid operand to INVEPT/INVVPID."
836	};
837	#endif /* VBOX_STRICT && LOG_ENABLED */
838
839
840	/**
841	* Get the CR0 guest/host mask that does not change through the lifetime of a VM.
842	*
843	* Any bit set in this mask is owned by the host/hypervisor and would cause a
844	* VM-exit when modified by the guest.
845	*
846	* @returns The static CR0 guest/host mask.
847	* @param pVCpu The cross context virtual CPU structure.
848	*/
849	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr0Mask(PCVMCPU pVCpu)
850	{
851	/*
852	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
853	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
854	*/
855	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
856	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
857	* and @bugref{6944}. */
858	PVM pVM = pVCpu->CTX_SUFF(pVM);
859	return ( X86_CR0_PE
860	\| X86_CR0_NE
861	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
862	\| X86_CR0_PG
863	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
864	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
865	\| X86_CR0_NW); /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
866	}
867
868
869	/**
870	* Gets the CR4 guest/host mask that does not change through the lifetime of a VM.
871	*
872	* Any bit set in this mask is owned by the host/hypervisor and would cause a
873	* VM-exit when modified by the guest.
874	*
875	* @returns The static CR4 guest/host mask.
876	* @param pVCpu The cross context virtual CPU structure.
877	*/
878	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr4Mask(PCVMCPU pVCpu)
879	{
880	/*
881	* We need to look at the host features here (for e.g. OSXSAVE, PCID) because
882	* these bits are reserved on hardware that does not support them. Since the
883	* CPU cannot refer to our virtual CPUID, we need to intercept CR4 changes to
884	* these bits and handle it depending on whether we expose them to the guest.
885	*/
886	PVM pVM = pVCpu->CTX_SUFF(pVM);
887	bool const fXSaveRstor = pVM->cpum.ro.HostFeatures.fXSaveRstor;
888	bool const fPcid = pVM->cpum.ro.HostFeatures.fPcid;
889	return ( X86_CR4_VMXE
890	\| X86_CR4_VME
891	\| X86_CR4_PAE
892	\| X86_CR4_PGE
893	\| X86_CR4_PSE
894	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
895	\| (fPcid ? X86_CR4_PCIDE : 0));
896	}
897
898
899	/**
900	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
901	* area.
902	*
903	* @returns @c true if it's different, @c false otherwise.
904	* @param pVmcsInfo The VMCS info. object.
905	*/
906	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
907	{
908	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
909	&& pVmcsInfo->pvGuestMsrStore);
910	}
911
912
913	/**
914	* Sets the given Processor-based VM-execution controls.
915	*
916	* @param pVmxTransient The VMX-transient structure.
917	* @param uProcCtls The Processor-based VM-execution controls to set.
918	*/
919	static void hmR0VmxSetProcCtlsVmcs(PVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
920	{
921	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
922	if ((pVmcsInfo->u32ProcCtls & uProcCtls) != uProcCtls)
923	{
924	pVmcsInfo->u32ProcCtls \|= uProcCtls;
925	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
926	AssertRC(rc);
927	}
928	}
929
930
931	/**
932	* Removes the given Processor-based VM-execution controls.
933	*
934	* @param pVCpu The cross context virtual CPU structure.
935	* @param pVmxTransient The VMX-transient structure.
936	* @param uProcCtls The Processor-based VM-execution controls to remove.
937	*
938	* @remarks When executing a nested-guest, this will not remove any of the specified
939	* controls if the guest hypervisor has set any one of them.
940	*/
941	static void hmR0VmxRemoveProcCtlsVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uProcCtls)
942	{
943	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
944	if (pVmcsInfo->u32ProcCtls & uProcCtls)
945	{
946	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
947	bool const fRemoveCtls = !pVmxTransient->fIsNestedGuest
948	? true
949	: !CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uProcCtls);
950	#else
951	NOREF(pVCpu);
952	bool const fRemoveCtls = true;
953	#endif
954	if (fRemoveCtls)
955	{
956	pVmcsInfo->u32ProcCtls &= ~uProcCtls;
957	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
958	AssertRC(rc);
959	}
960	}
961	}
962
963
964	/**
965	* Sets the TSC offset for the current VMCS.
966	*
967	* @param pVCpu The cross context virtual CPU structure.
968	* @param uTscOffset The TSC offset to set.
969	* @param pVmcsInfo The VMCS info. object.
970	*/
971	static void hmR0VmxSetTscOffsetVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t uTscOffset)
972	{
973	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
974	if (pVmcsInfo->u64TscOffset != uTscOffset)
975	{
976	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
977	AssertRC(rc);
978	pVmcsInfo->u64TscOffset = uTscOffset;
979	}
980	}
981
982
983	/**
984	* Adds one or more exceptions to the exception bitmap and commits it to the current
985	* VMCS.
986	*
987	* @returns VBox status code.
988	* @param pVmxTransient The VMX-transient structure.
989	* @param uXcptMask The exception(s) to add.
990	*/
991	static int hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
992	{
993	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
994	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
995	if ((uXcptBitmap & uXcptMask) != uXcptMask)
996	{
997	uXcptBitmap \|= uXcptMask;
998	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
999	AssertRCReturn(rc, rc);
1000	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
1001	}
1002	return VINF_SUCCESS;
1003	}
1004
1005
1006	/**
1007	* Adds an exception to the exception bitmap and commits it to the current VMCS.
1008	*
1009	* @returns VBox status code.
1010	* @param pVmxTransient The VMX-transient structure.
1011	* @param uXcpt The exception to add.
1012	*/
1013	static int hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1014	{
1015	Assert(uXcpt <= X86_XCPT_LAST);
1016	return hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
1017	}
1018
1019
1020	/**
1021	* Remove one or more exceptions from the exception bitmap and commits it to the
1022	* current VMCS.
1023	*
1024	* This takes care of not removing the exception intercept if a nested-guest
1025	* requires the exception to be intercepted.
1026	*
1027	* @returns VBox status code.
1028	* @param pVCpu The cross context virtual CPU structure.
1029	* @param pVmxTransient The VMX-transient structure.
1030	* @param uXcptMask The exception(s) to remove.
1031	*/
1032	static int hmR0VmxRemoveXcptInterceptMask(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
1033	{
1034	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1035	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
1036	if (u32XcptBitmap & uXcptMask)
1037	{
1038	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1039	if (!pVmxTransient->fIsNestedGuest)
1040	{ /* likely */ }
1041	else
1042	{
1043	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
1044	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
1045	}
1046	#endif
1047	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
1048	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
1049	\| RT_BIT(X86_XCPT_DE)
1050	\| RT_BIT(X86_XCPT_NM)
1051	\| RT_BIT(X86_XCPT_TS)
1052	\| RT_BIT(X86_XCPT_UD)
1053	\| RT_BIT(X86_XCPT_NP)
1054	\| RT_BIT(X86_XCPT_SS)
1055	\| RT_BIT(X86_XCPT_GP)
1056	\| RT_BIT(X86_XCPT_PF)
1057	\| RT_BIT(X86_XCPT_MF));
1058	#elif defined(HMVMX_ALWAYS_TRAP_PF)
1059	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
1060	#endif
1061	if (uXcptMask)
1062	{
1063	/* Validate we are not removing any essential exception intercepts. */
1064	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF))); RT_NOREF(pVCpu);
1065	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
1066	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
1067
1068	/* Remove it from the exception bitmap. */
1069	u32XcptBitmap &= ~uXcptMask;
1070
1071	/* Commit and update the cache if necessary. */
1072	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
1073	{
1074	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1075	AssertRCReturn(rc, rc);
1076	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
1077	}
1078	}
1079	}
1080	return VINF_SUCCESS;
1081	}
1082
1083
1084	/**
1085	* Remove an exceptions from the exception bitmap and commits it to the current
1086	* VMCS.
1087	*
1088	* @returns VBox status code.
1089	* @param pVCpu The cross context virtual CPU structure.
1090	* @param pVmxTransient The VMX-transient structure.
1091	* @param uXcpt The exception to remove.
1092	*/
1093	static int hmR0VmxRemoveXcptIntercept(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
1094	{
1095	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
1096	}
1097
1098
1099	/**
1100	* Loads the VMCS specified by the VMCS info. object.
1101	*
1102	* @returns VBox status code.
1103	* @param pVmcsInfo The VMCS info. object.
1104	*
1105	* @remarks Can be called with interrupts disabled.
1106	*/
1107	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
1108	{
1109	Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
1110	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1111
1112	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
1113	if (RT_SUCCESS(rc))
1114	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1115	return rc;
1116	}
1117
1118
1119	/**
1120	* Clears the VMCS specified by the VMCS info. object.
1121	*
1122	* @returns VBox status code.
1123	* @param pVmcsInfo The VMCS info. object.
1124	*
1125	* @remarks Can be called with interrupts disabled.
1126	*/
1127	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
1128	{
1129	Assert(pVmcsInfo->HCPhysVmcs != 0 && pVmcsInfo->HCPhysVmcs != NIL_RTHCPHYS);
1130	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1131
1132	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
1133	if (RT_SUCCESS(rc))
1134	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1135	return rc;
1136	}
1137
1138
1139	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1140	/**
1141	* Loads the shadow VMCS specified by the VMCS info. object.
1142	*
1143	* @returns VBox status code.
1144	* @param pVmcsInfo The VMCS info. object.
1145	*
1146	* @remarks Can be called with interrupts disabled.
1147	*/
1148	static int hmR0VmxLoadShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1149	{
1150	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1151	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1152
1153	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysShadowVmcs);
1154	if (RT_SUCCESS(rc))
1155	pVmcsInfo->fShadowVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
1156	return rc;
1157	}
1158
1159
1160	/**
1161	* Clears the shadow VMCS specified by the VMCS info. object.
1162	*
1163	* @returns VBox status code.
1164	* @param pVmcsInfo The VMCS info. object.
1165	*
1166	* @remarks Can be called with interrupts disabled.
1167	*/
1168	static int hmR0VmxClearShadowVmcs(PVMXVMCSINFO pVmcsInfo)
1169	{
1170	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1171	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
1172
1173	int rc = VMXClearVmcs(pVmcsInfo->HCPhysShadowVmcs);
1174	if (RT_SUCCESS(rc))
1175	pVmcsInfo->fShadowVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
1176	return rc;
1177	}
1178
1179
1180	/**
1181	* Switches from and to the specified VMCSes.
1182	*
1183	* @returns VBox status code.
1184	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
1185	* @param pVmcsInfoTo The VMCS info. object we are switching to.
1186	*
1187	* @remarks Called with interrupts disabled.
1188	*/
1189	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
1190	{
1191	/*
1192	* Clear the VMCS we are switching out if it has not already been cleared.
1193	* This will sync any CPU internal data back to the VMCS.
1194	*/
1195	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1196	{
1197	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
1198	if (RT_SUCCESS(rc))
1199	{
1200	/*
1201	* The shadow VMCS, if any, would not be active at this point since we
1202	* would have cleared it while importing the virtual hardware-virtualization
1203	* state as part the VMLAUNCH/VMRESUME VM-exit. Hence, there's no need to
1204	* clear the shadow VMCS here, just assert for safety.
1205	*/
1206	Assert(!pVmcsInfoFrom->pvShadowVmcs \|\| pVmcsInfoFrom->fShadowVmcsState == VMX_V_VMCS_LAUNCH_STATE_CLEAR);
1207	}
1208	else
1209	return rc;
1210	}
1211
1212	/*
1213	* Clear the VMCS we are switching to if it has not already been cleared.
1214	* This will initialize the VMCS launch state to "clear" required for loading it.
1215	*
1216	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
1217	*/
1218	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1219	{
1220	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
1221	if (RT_SUCCESS(rc))
1222	{ /* likely */ }
1223	else
1224	return rc;
1225	}
1226
1227	/*
1228	* Finally, load the VMCS we are switching to.
1229	*/
1230	return hmR0VmxLoadVmcs(pVmcsInfoTo);
1231	}
1232
1233
1234	/**
1235	* Switches between the guest VMCS and the nested-guest VMCS as specified by the
1236	* caller.
1237	*
1238	* @returns VBox status code.
1239	* @param pVCpu The cross context virtual CPU structure.
1240	* @param fSwitchToNstGstVmcs Whether to switch to the nested-guest VMCS (pass
1241	* true) or guest VMCS (pass false).
1242	*/
1243	static int hmR0VmxSwitchToGstOrNstGstVmcs(PVMCPU pVCpu, bool fSwitchToNstGstVmcs)
1244	{
1245	/* Ensure we have synced everything from the guest-CPU context to the VMCS before switching. */
1246	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
1247
1248	PVMXVMCSINFO pVmcsInfoFrom;
1249	PVMXVMCSINFO pVmcsInfoTo;
1250	if (fSwitchToNstGstVmcs)
1251	{
1252	pVmcsInfoFrom = &pVCpu->hm.s.vmx.VmcsInfo;
1253	pVmcsInfoTo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1254	}
1255	else
1256	{
1257	pVmcsInfoFrom = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1258	pVmcsInfoTo = &pVCpu->hm.s.vmx.VmcsInfo;
1259	}
1260
1261	/*
1262	* Disable interrupts to prevent being preempted while we switch the current VMCS as the
1263	* preemption hook code path acquires the current VMCS.
1264	*/
1265	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1266
1267	int rc = hmR0VmxSwitchVmcs(pVmcsInfoFrom, pVmcsInfoTo);
1268	if (RT_SUCCESS(rc))
1269	{
1270	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fSwitchToNstGstVmcs;
1271
1272	/*
1273	* If we are switching to a VMCS that was executed on a different host CPU or was
1274	* never executed before, flag that we need to export the host state before executing
1275	* guest/nested-guest code using hardware-assisted VMX.
1276	*
1277	* This could probably be done in a preemptible context since the preemption hook
1278	* will flag the necessary change in host context. However, since preemption is
1279	* already disabled and to avoid making assumptions about host specific code in
1280	* RTMpCpuId when called with preemption enabled, we'll do this while preemption is
1281	* disabled.
1282	*/
1283	if (pVmcsInfoTo->idHostCpu == RTMpCpuId())
1284	{ /* likely */ }
1285	else
1286	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE);
1287
1288	ASMSetFlags(fEFlags);
1289
1290	/*
1291	* We use a different VM-exit MSR-store areas for the guest and nested-guest. Hence,
1292	* flag that we need to update the host MSR values there. Even if we decide in the
1293	* future to share the VM-exit MSR-store area page between the guest and nested-guest,
1294	* if its content differs, we would have to update the host MSRs anyway.
1295	*/
1296	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
1297	}
1298	else
1299	ASMSetFlags(fEFlags);
1300	return rc;
1301	}
1302	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
1303
1304
1305	/**
1306	* Updates the VM's last error record.
1307	*
1308	* If there was a VMX instruction error, reads the error data from the VMCS and
1309	* updates VCPU's last error record as well.
1310	*
1311	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1312	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
1313	* VERR_VMX_INVALID_VMCS_FIELD.
1314	* @param rc The error code.
1315	*/
1316	static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
1317	{
1318	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
1319	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
1320	{
1321	AssertPtrReturnVoid(pVCpu);
1322	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
1323	}
1324	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
1325	}
1326
1327
1328	#ifdef VBOX_STRICT
1329	/**
1330	* Reads the VM-entry interruption-information field from the VMCS into the VMX
1331	* transient structure.
1332	*
1333	* @returns VBox status code.
1334	* @param pVmxTransient The VMX-transient structure.
1335	*
1336	* @remarks No-long-jump zone!!!
1337	*/
1338	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1339	{
1340	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
1341	AssertRCReturn(rc, rc);
1342	return VINF_SUCCESS;
1343	}
1344
1345
1346	/**
1347	* Reads the VM-entry exception error code field from the VMCS into
1348	* the VMX transient structure.
1349	*
1350	* @returns VBox status code.
1351	* @param pVmxTransient The VMX-transient structure.
1352	*
1353	* @remarks No-long-jump zone!!!
1354	*/
1355	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1356	{
1357	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
1358	AssertRCReturn(rc, rc);
1359	return VINF_SUCCESS;
1360	}
1361
1362
1363	/**
1364	* Reads the VM-entry exception error code field from the VMCS into
1365	* the VMX transient structure.
1366	*
1367	* @returns VBox status code.
1368	* @param pVmxTransient The VMX-transient structure.
1369	*
1370	* @remarks No-long-jump zone!!!
1371	*/
1372	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1373	{
1374	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
1375	AssertRCReturn(rc, rc);
1376	return VINF_SUCCESS;
1377	}
1378	#endif /* VBOX_STRICT */
1379
1380
1381	/**
1382	* Reads the VM-exit interruption-information field from the VMCS into the VMX
1383	* transient structure.
1384	*
1385	* @returns VBox status code.
1386	* @param pVmxTransient The VMX-transient structure.
1387	*/
1388	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
1389	{
1390	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
1391	{
1392	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1393	AssertRCReturn(rc,rc);
1394	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
1395	}
1396	return VINF_SUCCESS;
1397	}
1398
1399
1400	/**
1401	* Reads the VM-exit interruption error code from the VMCS into the VMX
1402	* transient structure.
1403	*
1404	* @returns VBox status code.
1405	* @param pVmxTransient The VMX-transient structure.
1406	*/
1407	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1408	{
1409	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
1410	{
1411	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1412	AssertRCReturn(rc, rc);
1413	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1414	}
1415	return VINF_SUCCESS;
1416	}
1417
1418
1419	/**
1420	* Reads the VM-exit instruction length field from the VMCS into the VMX
1421	* transient structure.
1422	*
1423	* @returns VBox status code.
1424	* @param pVmxTransient The VMX-transient structure.
1425	*/
1426	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1427	{
1428	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1429	{
1430	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
1431	AssertRCReturn(rc, rc);
1432	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1433	}
1434	return VINF_SUCCESS;
1435	}
1436
1437
1438	/**
1439	* Reads the VM-exit instruction-information field from the VMCS into
1440	* the VMX transient structure.
1441	*
1442	* @returns VBox status code.
1443	* @param pVmxTransient The VMX-transient structure.
1444	*/
1445	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1446	{
1447	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1448	{
1449	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1450	AssertRCReturn(rc, rc);
1451	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1452	}
1453	return VINF_SUCCESS;
1454	}
1455
1456
1457	/**
1458	* Reads the Exit Qualification from the VMCS into the VMX transient structure.
1459	*
1460	* @returns VBox status code.
1461	* @param pVCpu The cross context virtual CPU structure of the
1462	* calling EMT. (Required for the VMCS cache case.)
1463	* @param pVmxTransient The VMX-transient structure.
1464	*/
1465	DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1466	{
1467	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1468	{
1469	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
1470	AssertRCReturn(rc, rc);
1471	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1472	}
1473	return VINF_SUCCESS;
1474	}
1475
1476
1477	/**
1478	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1479	*
1480	* @returns VBox status code.
1481	* @param pVCpu The cross context virtual CPU structure of the
1482	* calling EMT. (Required for the VMCS cache case.)
1483	* @param pVmxTransient The VMX-transient structure.
1484	*/
1485	DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1486	{
1487	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1488	{
1489	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); NOREF(pVCpu);
1490	AssertRCReturn(rc, rc);
1491	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1492	}
1493	return VINF_SUCCESS;
1494	}
1495
1496
1497	/**
1498	* Reads the Guest-physical address from the VMCS into the VMX transient structure.
1499	*
1500	* @returns VBox status code.
1501	* @param pVCpu The cross context virtual CPU structure of the
1502	* calling EMT. (Required for the VMCS cache case.)
1503	* @param pVmxTransient The VMX-transient structure.
1504	*/
1505	DECLINLINE(int) hmR0VmxReadGuestPhysicalAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1506	{
1507	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_PHYSICAL_ADDR))
1508	{
1509	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr); NOREF(pVCpu);
1510	AssertRCReturn(rc, rc);
1511	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_PHYSICAL_ADDR;
1512	}
1513	return VINF_SUCCESS;
1514	}
1515
1516
1517	/**
1518	* Reads the IDT-vectoring information field from the VMCS into the VMX
1519	* transient structure.
1520	*
1521	* @returns VBox status code.
1522	* @param pVmxTransient The VMX-transient structure.
1523	*
1524	* @remarks No-long-jump zone!!!
1525	*/
1526	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1527	{
1528	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1529	{
1530	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1531	AssertRCReturn(rc, rc);
1532	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1533	}
1534	return VINF_SUCCESS;
1535	}
1536
1537
1538	/**
1539	* Reads the IDT-vectoring error code from the VMCS into the VMX
1540	* transient structure.
1541	*
1542	* @returns VBox status code.
1543	* @param pVmxTransient The VMX-transient structure.
1544	*/
1545	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1546	{
1547	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1548	{
1549	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1550	AssertRCReturn(rc, rc);
1551	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1552	}
1553	return VINF_SUCCESS;
1554	}
1555
1556	#ifdef HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
1557	/**
1558	* Reads all relevant read-only VMCS fields into the VMX transient structure.
1559	*
1560	* @returns VBox status code.
1561	* @param pVCpu The cross context virtual CPU structure of the
1562	* calling EMT. (Required for the VMCS cache case.)
1563	* @param pVmxTransient The VMX-transient structure.
1564	*/
1565	static int hmR0VmxReadAllRoFieldsVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1566	{
1567	NOREF(pVCpu); /* Used implicitly by VMXReadVmcsGstN on 32-bit hosts. */
1568	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual);
1569	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
1570	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1571	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1572	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1573	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1574	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1575	rc \|= VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr);
1576	rc \|= VMXReadVmcsGstN(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr);
1577	AssertRCReturn(rc, rc);
1578	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION
1579	\| HMVMX_READ_EXIT_INSTR_LEN
1580	\| HMVMX_READ_EXIT_INSTR_INFO
1581	\| HMVMX_READ_IDT_VECTORING_INFO
1582	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE
1583	\| HMVMX_READ_EXIT_INTERRUPTION_INFO
1584	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
1585	\| HMVMX_READ_GUEST_LINEAR_ADDR
1586	\| HMVMX_READ_GUEST_PHYSICAL_ADDR;
1587	return VINF_SUCCESS;
1588	}
1589	#endif
1590
1591	/**
1592	* Enters VMX root mode operation on the current CPU.
1593	*
1594	* @returns VBox status code.
1595	* @param pVM The cross context VM structure. Can be
1596	* NULL, after a resume.
1597	* @param HCPhysCpuPage Physical address of the VMXON region.
1598	* @param pvCpuPage Pointer to the VMXON region.
1599	*/
1600	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1601	{
1602	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1603	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1604	Assert(pvCpuPage);
1605	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1606
1607	if (pVM)
1608	{
1609	/* Write the VMCS revision identifier to the VMXON region. */
1610	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1611	}
1612
1613	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1614	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1615
1616	/* Enable the VMX bit in CR4 if necessary. */
1617	RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1618
1619	/* Enter VMX root mode. */
1620	int rc = VMXEnable(HCPhysCpuPage);
1621	if (RT_FAILURE(rc))
1622	{
1623	if (!(uOldCr4 & X86_CR4_VMXE))
1624	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1625
1626	if (pVM)
1627	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1628	}
1629
1630	/* Restore interrupts. */
1631	ASMSetFlags(fEFlags);
1632	return rc;
1633	}
1634
1635
1636	/**
1637	* Exits VMX root mode operation on the current CPU.
1638	*
1639	* @returns VBox status code.
1640	*/
1641	static int hmR0VmxLeaveRootMode(void)
1642	{
1643	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1644
1645	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1646	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1647
1648	/* If we're for some reason not in VMX root mode, then don't leave it. */
1649	RTCCUINTREG const uHostCr4 = ASMGetCR4();
1650
1651	int rc;
1652	if (uHostCr4 & X86_CR4_VMXE)
1653	{
1654	/* Exit VMX root mode and clear the VMX bit in CR4. */
1655	VMXDisable();
1656	SUPR0ChangeCR4(0 /* fOrMask */, ~X86_CR4_VMXE);
1657	rc = VINF_SUCCESS;
1658	}
1659	else
1660	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1661
1662	/* Restore interrupts. */
1663	ASMSetFlags(fEFlags);
1664	return rc;
1665	}
1666
1667
1668	/**
1669	* Allocates and maps a physically contiguous page. The allocated page is
1670	* zero'd out (used by various VT-x structures).
1671	*
1672	* @returns IPRT status code.
1673	* @param pMemObj Pointer to the ring-0 memory object.
1674	* @param ppVirt Where to store the virtual address of the allocation.
1675	* @param pHCPhys Where to store the physical address of the allocation.
1676	*/
1677	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1678	{
1679	AssertPtr(pMemObj);
1680	AssertPtr(ppVirt);
1681	AssertPtr(pHCPhys);
1682	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1683	if (RT_FAILURE(rc))
1684	return rc;
1685	ppVirt = RTR0MemObjAddress(pMemObj);
1686	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1687	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1688	return VINF_SUCCESS;
1689	}
1690
1691
1692	/**
1693	* Frees and unmaps an allocated, physical page.
1694	*
1695	* @param pMemObj Pointer to the ring-0 memory object.
1696	* @param ppVirt Where to re-initialize the virtual address of allocation as
1697	* 0.
1698	* @param pHCPhys Where to re-initialize the physical address of the
1699	* allocation as 0.
1700	*/
1701	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1702	{
1703	AssertPtr(pMemObj);
1704	AssertPtr(ppVirt);
1705	AssertPtr(pHCPhys);
1706	/* NULL is valid, accepted and ignored by the free function below. */
1707	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1708	*pMemObj = NIL_RTR0MEMOBJ;
1709	*ppVirt = NULL;
1710	*pHCPhys = NIL_RTHCPHYS;
1711	}
1712
1713
1714	/**
1715	* Initializes a VMCS info. object.
1716	*
1717	* @param pVmcsInfo The VMCS info. object.
1718	*/
1719	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1720	{
1721	memset(pVmcsInfo, 0, sizeof(*pVmcsInfo));
1722
1723	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1724	Assert(pVmcsInfo->hMemObjShadowVmcs == NIL_RTR0MEMOBJ);
1725	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1726	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1727	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1728	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1729	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1730	pVmcsInfo->HCPhysShadowVmcs = NIL_RTHCPHYS;
1731	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1732	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1733	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1734	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1735	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1736	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1737	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1738	pVmcsInfo->idHostCpu = NIL_RTCPUID;
1739	}
1740
1741
1742	/**
1743	* Frees the VT-x structures for a VMCS info. object.
1744	*
1745	* @param pVM The cross context VM structure.
1746	* @param pVmcsInfo The VMCS info. object.
1747	*/
1748	static void hmR0VmxFreeVmcsInfo(PVM pVM, PVMXVMCSINFO pVmcsInfo)
1749	{
1750	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1751
1752	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1753	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1754	hmR0VmxPageFree(&pVmcsInfo->hMemObjShadowVmcs, &pVmcsInfo->pvShadowVmcs, &pVmcsInfo->HCPhysShadowVmcs);
1755	#endif
1756
1757	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1758	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1759
1760	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1761	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1762	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1763
1764	hmR0VmxInitVmcsInfo(pVmcsInfo);
1765	}
1766
1767
1768	/**
1769	* Allocates the VT-x structures for a VMCS info. object.
1770	*
1771	* @returns VBox status code.
1772	* @param pVCpu The cross context virtual CPU structure.
1773	* @param pVmcsInfo The VMCS info. object.
1774	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1775	*/
1776	static int hmR0VmxAllocVmcsInfo(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1777	{
1778	PVM pVM = pVCpu->CTX_SUFF(pVM);
1779
1780	/* Allocate the guest VM control structure (VMCS). */
1781	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1782	if (RT_SUCCESS(rc))
1783	{
1784	if (!fIsNstGstVmcs)
1785	{
1786	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1787	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1788	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjShadowVmcs, &pVmcsInfo->pvShadowVmcs, &pVmcsInfo->HCPhysShadowVmcs);
1789	#endif
1790	if (RT_SUCCESS(rc))
1791	{
1792	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1793	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1794	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1795	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic, NULL /pR3Ptr/);
1796	}
1797	}
1798	else
1799	{
1800	/* We don't yet support exposing VMCS shadowing to the guest. */
1801	Assert(pVmcsInfo->HCPhysShadowVmcs == NIL_RTHCPHYS);
1802	Assert(!pVmcsInfo->pvShadowVmcs);
1803
1804	/* Get the allocated virtual-APIC page from CPUM. */
1805	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
1806	{
1807	/** @todo NSTVMX: Get rid of this. There is no need to allocate a separate HC
1808	* page for this. Use the one provided by the nested-guest directly. */
1809	pVmcsInfo->pbVirtApic = (uint8_t *)CPUMGetGuestVmxVirtApicPage(pVCpu, &pVCpu->cpum.GstCtx,
1810	&pVmcsInfo->HCPhysVirtApic);
1811	Assert(pVmcsInfo->pbVirtApic);
1812	Assert(pVmcsInfo->HCPhysVirtApic && pVmcsInfo->HCPhysVirtApic != NIL_RTHCPHYS);
1813	}
1814	}
1815
1816	if (RT_SUCCESS(rc))
1817	{
1818	/*
1819	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1820	* transparent accesses of specific MSRs.
1821	*
1822	* If the condition for enabling MSR bitmaps changes here, don't forget to
1823	* update HMIsMsrBitmapActive().
1824	*
1825	* We don't share MSR bitmaps between the guest and nested-guest as we then
1826	* don't need to care about carefully restoring the guest MSR bitmap.
1827	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1828	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1829	* We also don't need to re-initialize the nested-guest MSR bitmap here as
1830	* we do that later while merging VMCS.
1831	*/
1832	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1833	{
1834	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1835	if ( RT_SUCCESS(rc)
1836	&& !fIsNstGstVmcs)
1837	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1838	}
1839
1840	if (RT_SUCCESS(rc))
1841	{
1842	/*
1843	* Allocate the VM-entry MSR-load area for the guest MSRs.
1844	*
1845	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1846	* the guest and nested-guest.
1847	*/
1848	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1849	&pVmcsInfo->HCPhysGuestMsrLoad);
1850	if (RT_SUCCESS(rc))
1851	{
1852	/*
1853	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1854	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1855	*/
1856	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1857	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1858	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1859
1860	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1861	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1862	&pVmcsInfo->HCPhysHostMsrLoad);
1863	}
1864	}
1865	}
1866	}
1867
1868	return rc;
1869	}
1870
1871
1872	/**
1873	* Free all VT-x structures for the VM.
1874	*
1875	* @returns IPRT status code.
1876	* @param pVM The cross context VM structure.
1877	*/
1878	static void hmR0VmxStructsFree(PVM pVM)
1879	{
1880	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1881	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1882	#endif
1883	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1884
1885	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1886	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1887	{
1888	RTMemFree(pVM->hm.s.vmx.paShadowVmcsFields);
1889	RTMemFree(pVM->hm.s.vmx.paShadowVmcsRoFields);
1890	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjVmreadBitmap, &pVM->hm.s.vmx.pvVmreadBitmap, &pVM->hm.s.vmx.HCPhysVmreadBitmap);
1891	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap, &pVM->hm.s.vmx.HCPhysVmwriteBitmap);
1892	}
1893	#endif
1894
1895	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1896	{
1897	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1898	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1899	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1900	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1901	if (pVM->cpum.ro.GuestFeatures.fVmx)
1902	{
1903	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1904	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1905	}
1906	#endif
1907	}
1908	}
1909
1910
1911	/**
1912	* Allocate all VT-x structures for the VM.
1913	*
1914	* @returns IPRT status code.
1915	* @param pVM The cross context VM structure.
1916	*/
1917	static int hmR0VmxStructsAlloc(PVM pVM)
1918	{
1919	/*
1920	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1921	* The VMCS size cannot be more than 4096 bytes.
1922	*
1923	* See Intel spec. Appendix A.1 "Basic VMX Information".
1924	*/
1925	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1926	if (cbVmcs <= X86_PAGE_4K_SIZE)
1927	{ /* likely */ }
1928	else
1929	{
1930	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1931	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1932	}
1933
1934	/*
1935	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
1936	*/
1937	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1938	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1939	Assert(pVM->hm.s.vmx.pbScratch == NULL);
1940	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1941	#endif
1942
1943	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1944	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1945	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1946
1947	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1948	{
1949	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1950	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1951	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1952	}
1953
1954	/*
1955	* Allocate per-VM VT-x structures.
1956	*/
1957	int rc = VINF_SUCCESS;
1958	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1959	/* Allocate crash-dump magic scratch page. */
1960	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1961	if (RT_FAILURE(rc))
1962	{
1963	hmR0VmxStructsFree(pVM);
1964	return rc;
1965	}
1966	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1967	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1968	#endif
1969
1970	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1971	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1972	{
1973	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1974	&pVM->hm.s.vmx.HCPhysApicAccess);
1975	if (RT_FAILURE(rc))
1976	{
1977	hmR0VmxStructsFree(pVM);
1978	return rc;
1979	}
1980	}
1981
1982	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1983	/* Allocate the shadow VMCS fields array, VMREAD, VMWRITE bitmaps.. */
1984	if (pVM->hm.s.vmx.fUseVmcsShadowing)
1985	{
1986	Assert(!pVM->hm.s.vmx.cShadowVmcsFields);
1987	Assert(!pVM->hm.s.vmx.cShadowVmcsRoFields);
1988	pVM->hm.s.vmx.paShadowVmcsFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
1989	pVM->hm.s.vmx.paShadowVmcsRoFields = (uint32_t *)RTMemAllocZ(sizeof(g_aVmcsFields));
1990	if (RT_LIKELY( pVM->hm.s.vmx.paShadowVmcsFields
1991	&& pVM->hm.s.vmx.paShadowVmcsRoFields))
1992	{
1993	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjVmreadBitmap, &pVM->hm.s.vmx.pvVmreadBitmap,
1994	&pVM->hm.s.vmx.HCPhysVmreadBitmap);
1995	if (RT_SUCCESS(rc))
1996	{
1997	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjVmwriteBitmap, &pVM->hm.s.vmx.pvVmwriteBitmap,
1998	&pVM->hm.s.vmx.HCPhysVmwriteBitmap);
1999	}
2000	}
2001	else
2002	rc = VERR_NO_MEMORY;
2003
2004	if (RT_FAILURE(rc))
2005	{
2006	hmR0VmxStructsFree(pVM);
2007	return rc;
2008	}
2009	}
2010	#endif
2011
2012	/*
2013	* Initialize per-VCPU VT-x structures.
2014	*/
2015	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
2016	{
2017	/* Allocate the guest VMCS structures. */
2018	PVMCPU pVCpu = &pVM->aCpus[idCpu];
2019	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
2020	if (RT_SUCCESS(rc))
2021	{
2022	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2023	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
2024	if (pVM->cpum.ro.GuestFeatures.fVmx)
2025	{
2026	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
2027	if (RT_SUCCESS(rc))
2028	{ /* likely */ }
2029	else
2030	break;
2031	}
2032	#endif
2033	}
2034	else
2035	break;
2036	}
2037
2038	if (RT_FAILURE(rc))
2039	{
2040	hmR0VmxStructsFree(pVM);
2041	return rc;
2042	}
2043
2044	return VINF_SUCCESS;
2045	}
2046
2047	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2048	/**
2049	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
2050	*
2051	* @returns @c true if the MSR is intercepted, @c false otherwise.
2052	* @param pvMsrBitmap The MSR bitmap.
2053	* @param offMsr The MSR byte offset.
2054	* @param iBit The bit offset from the byte offset.
2055	*/
2056	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
2057	{
2058	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
2059	Assert(pbMsrBitmap);
2060	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
2061	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
2062	}
2063	#endif
2064
2065	/**
2066	* Sets the permission bits for the specified MSR in the given MSR bitmap.
2067	*
2068	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
2069	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
2070	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
2071	* the read/write access of this MSR.
2072	*
2073	* @param pVCpu The cross context virtual CPU structure.
2074	* @param pVmcsInfo The VMCS info. object.
2075	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2076	* @param idMsr The MSR value.
2077	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
2078	* include both a read -and- a write permission!
2079	*
2080	* @sa CPUMGetVmxMsrPermission.
2081	* @remarks Can be called with interrupts disabled.
2082	*/
2083	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
2084	{
2085	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
2086	Assert(pbMsrBitmap);
2087	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
2088
2089	/*
2090	* MSR-bitmap Layout:
2091	* Byte index MSR range Interpreted as
2092	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
2093	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
2094	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
2095	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
2096	*
2097	* A bit corresponding to an MSR within the above range causes a VM-exit
2098	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
2099	* the MSR range, it always cause a VM-exit.
2100	*
2101	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
2102	*/
2103	uint16_t const offBitmapRead = 0;
2104	uint16_t const offBitmapWrite = 0x800;
2105	uint16_t offMsr;
2106	int32_t iBit;
2107	if (idMsr <= UINT32_C(0x00001fff))
2108	{
2109	offMsr = 0;
2110	iBit = idMsr;
2111	}
2112	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
2113	{
2114	offMsr = 0x400;
2115	iBit = idMsr - UINT32_C(0xc0000000);
2116	}
2117	else
2118	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
2119
2120	/*
2121	* Set the MSR read permission.
2122	*/
2123	uint16_t const offMsrRead = offBitmapRead + offMsr;
2124	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
2125	if (fMsrpm & VMXMSRPM_ALLOW_RD)
2126	{
2127	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2128	bool const fClear = !fIsNstGstVmcs ? true
2129	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
2130	#else
2131	RT_NOREF2(pVCpu, fIsNstGstVmcs);
2132	bool const fClear = true;
2133	#endif
2134	if (fClear)
2135	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
2136	}
2137	else
2138	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
2139
2140	/*
2141	* Set the MSR write permission.
2142	*/
2143	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
2144	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
2145	if (fMsrpm & VMXMSRPM_ALLOW_WR)
2146	{
2147	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2148	bool const fClear = !fIsNstGstVmcs ? true
2149	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
2150	#else
2151	RT_NOREF2(pVCpu, fIsNstGstVmcs);
2152	bool const fClear = true;
2153	#endif
2154	if (fClear)
2155	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
2156	}
2157	else
2158	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
2159	}
2160
2161
2162	/**
2163	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
2164	* area.
2165	*
2166	* @returns VBox status code.
2167	* @param pVCpu The cross context virtual CPU structure.
2168	* @param pVmcsInfo The VMCS info. object.
2169	* @param cMsrs The number of MSRs.
2170	*/
2171	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
2172	{
2173	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2174	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
2175	if (RT_LIKELY(cMsrs < cMaxSupportedMsrs))
2176	{
2177	/* Commit the MSR counts to the VMCS and update the cache. */
2178	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
2179	{
2180	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
2181	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
2182	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
2183	AssertRCReturn(rc, rc);
2184
2185	pVmcsInfo->cEntryMsrLoad = cMsrs;
2186	pVmcsInfo->cExitMsrStore = cMsrs;
2187	pVmcsInfo->cExitMsrLoad = cMsrs;
2188	}
2189	return VINF_SUCCESS;
2190	}
2191
2192	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u MaxSupported=%u\n", cMsrs, cMaxSupportedMsrs));
2193	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
2194	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2195	}
2196
2197
2198	/**
2199	* Adds a new (or updates the value of an existing) guest/host MSR
2200	* pair to be swapped during the world-switch as part of the
2201	* auto-load/store MSR area in the VMCS.
2202	*
2203	* @returns VBox status code.
2204	* @param pVCpu The cross context virtual CPU structure.
2205	* @param pVmxTransient The VMX-transient structure.
2206	* @param idMsr The MSR.
2207	* @param uGuestMsrValue Value of the guest MSR.
2208	* @param fSetReadWrite Whether to set the guest read/write access of this
2209	* MSR (thus not causing a VM-exit).
2210	* @param fUpdateHostMsr Whether to update the value of the host MSR if
2211	* necessary.
2212	*/
2213	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
2214	bool fSetReadWrite, bool fUpdateHostMsr)
2215	{
2216	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2217	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2218	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2219	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2220	uint32_t i;
2221
2222	/* Paranoia. */
2223	Assert(pGuestMsrLoad);
2224
2225	LogFlowFunc(("pVCpu=%p idMsr=%#RX32 uGestMsrValue=%#RX64\n", pVCpu, idMsr, uGuestMsrValue));
2226
2227	/* Check if the MSR already exists in the VM-entry MSR-load area. */
2228	for (i = 0; i < cMsrs; i++)
2229	{
2230	if (pGuestMsrLoad[i].u32Msr == idMsr)
2231	break;
2232	}
2233
2234	bool fAdded = false;
2235	if (i == cMsrs)
2236	{
2237	/* The MSR does not exist, bump the MSR count to make room for the new MSR. */
2238	++cMsrs;
2239	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2240	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
2241
2242	/* Set the guest to read/write this MSR without causing VM-exits. */
2243	if ( fSetReadWrite
2244	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
2245	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD_WR);
2246
2247	Log4Func(("Added MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
2248	fAdded = true;
2249	}
2250
2251	/* Update the MSR value for the newly added or already existing MSR. */
2252	pGuestMsrLoad[i].u32Msr = idMsr;
2253	pGuestMsrLoad[i].u64Value = uGuestMsrValue;
2254
2255	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
2256	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2257	{
2258	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2259	pGuestMsrStore[i].u32Msr = idMsr;
2260	pGuestMsrStore[i].u64Value = uGuestMsrValue;
2261	}
2262
2263	/* Update the corresponding slot in the host MSR area. */
2264	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2265	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad);
2266	Assert(pHostMsr != pVmcsInfo->pvGuestMsrStore);
2267	pHostMsr[i].u32Msr = idMsr;
2268
2269	/*
2270	* Only if the caller requests to update the host MSR value AND we've newly added the
2271	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
2272	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
2273	*
2274	* We do this for performance reasons since reading MSRs may be quite expensive.
2275	*/
2276	if (fAdded)
2277	{
2278	if (fUpdateHostMsr)
2279	{
2280	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2281	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2282	pHostMsr[i].u64Value = ASMRdMsr(idMsr);
2283	}
2284	else
2285	{
2286	/* Someone else can do the work. */
2287	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
2288	}
2289	}
2290	return VINF_SUCCESS;
2291	}
2292
2293
2294	/**
2295	* Removes a guest/host MSR pair to be swapped during the world-switch from the
2296	* auto-load/store MSR area in the VMCS.
2297	*
2298	* @returns VBox status code.
2299	* @param pVCpu The cross context virtual CPU structure.
2300	* @param pVmxTransient The VMX-transient structure.
2301	* @param idMsr The MSR.
2302	*/
2303	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
2304	{
2305	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2306	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
2307	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2308	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
2309
2310	LogFlowFunc(("pVCpu=%p idMsr=%#RX32\n", pVCpu, idMsr));
2311
2312	for (uint32_t i = 0; i < cMsrs; i++)
2313	{
2314	/* Find the MSR. */
2315	if (pGuestMsrLoad[i].u32Msr == idMsr)
2316	{
2317	/*
2318	* If it's the last MSR, we only need to reduce the MSR count.
2319	* If it's -not- the last MSR, copy the last MSR in place of it and reduce the MSR count.
2320	*/
2321	if (i < cMsrs - 1)
2322	{
2323	/* Remove it from the VM-entry MSR-load area. */
2324	pGuestMsrLoad[i].u32Msr = pGuestMsrLoad[cMsrs - 1].u32Msr;
2325	pGuestMsrLoad[i].u64Value = pGuestMsrLoad[cMsrs - 1].u64Value;
2326
2327	/* Remove it from the VM-exit MSR-store area if it's in a different page. */
2328	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
2329	{
2330	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2331	Assert(pGuestMsrStore[i].u32Msr == idMsr);
2332	pGuestMsrStore[i].u32Msr = pGuestMsrStore[cMsrs - 1].u32Msr;
2333	pGuestMsrStore[i].u64Value = pGuestMsrStore[cMsrs - 1].u64Value;
2334	}
2335
2336	/* Remove it from the VM-exit MSR-load area. */
2337	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2338	Assert(pHostMsr[i].u32Msr == idMsr);
2339	pHostMsr[i].u32Msr = pHostMsr[cMsrs - 1].u32Msr;
2340	pHostMsr[i].u64Value = pHostMsr[cMsrs - 1].u64Value;
2341	}
2342
2343	/* Reduce the count to reflect the removed MSR and bail. */
2344	--cMsrs;
2345	break;
2346	}
2347	}
2348
2349	/* Update the VMCS if the count changed (meaning the MSR was found and removed). */
2350	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
2351	{
2352	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
2353	AssertRCReturn(rc, rc);
2354
2355	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
2356	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2357	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
2358
2359	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
2360	return VINF_SUCCESS;
2361	}
2362
2363	return VERR_NOT_FOUND;
2364	}
2365
2366
2367	/**
2368	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
2369	*
2370	* @returns @c true if found, @c false otherwise.
2371	* @param pVmcsInfo The VMCS info. object.
2372	* @param idMsr The MSR to find.
2373	*/
2374	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
2375	{
2376	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2377	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
2378	Assert(pMsrs);
2379	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
2380	for (uint32_t i = 0; i < cMsrs; i++)
2381	{
2382	if (pMsrs[i].u32Msr == idMsr)
2383	return true;
2384	}
2385	return false;
2386	}
2387
2388
2389	/**
2390	* Updates the value of all host MSRs in the VM-exit MSR-load area.
2391	*
2392	* @param pVCpu The cross context virtual CPU structure.
2393	* @param pVmcsInfo The VMCS info. object.
2394	*
2395	* @remarks No-long-jump zone!!!
2396	*/
2397	static void hmR0VmxUpdateAutoLoadHostMsrs(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2398	{
2399	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2400
2401	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2402	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
2403	Assert(pHostMsrLoad);
2404	Assert(sizeof(pHostMsrLoad) cMsrs <= X86_PAGE_4K_SIZE);
2405	LogFlowFunc(("pVCpu=%p cMsrs=%u\n", pVCpu, cMsrs));
2406	for (uint32_t i = 0; i < cMsrs; i++)
2407	{
2408	/*
2409	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
2410	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
2411	*/
2412	if (pHostMsrLoad[i].u32Msr == MSR_K6_EFER)
2413	pHostMsrLoad[i].u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2414	else
2415	pHostMsrLoad[i].u64Value = ASMRdMsr(pHostMsrLoad[i].u32Msr);
2416	}
2417	}
2418
2419
2420	/**
2421	* Saves a set of host MSRs to allow read/write passthru access to the guest and
2422	* perform lazy restoration of the host MSRs while leaving VT-x.
2423	*
2424	* @param pVCpu The cross context virtual CPU structure.
2425	*
2426	* @remarks No-long-jump zone!!!
2427	*/
2428	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
2429	{
2430	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2431
2432	/*
2433	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
2434	*/
2435	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
2436	{
2437	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
2438	#if HC_ARCH_BITS == 64
2439	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2440	{
2441	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
2442	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
2443	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
2444	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
2445	}
2446	#endif
2447	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
2448	}
2449	}
2450
2451
2452	/**
2453	* Checks whether the MSR belongs to the set of guest MSRs that we restore
2454	* lazily while leaving VT-x.
2455	*
2456	* @returns true if it does, false otherwise.
2457	* @param pVCpu The cross context virtual CPU structure.
2458	* @param idMsr The MSR to check.
2459	*/
2460	static bool hmR0VmxIsLazyGuestMsr(PCVMCPU pVCpu, uint32_t idMsr)
2461	{
2462	NOREF(pVCpu);
2463	#if HC_ARCH_BITS == 64
2464	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2465	{
2466	switch (idMsr)
2467	{
2468	case MSR_K8_LSTAR:
2469	case MSR_K6_STAR:
2470	case MSR_K8_SF_MASK:
2471	case MSR_K8_KERNEL_GS_BASE:
2472	return true;
2473	}
2474	}
2475	#else
2476	RT_NOREF(pVCpu, idMsr);
2477	#endif
2478	return false;
2479	}
2480
2481
2482	/**
2483	* Loads a set of guests MSRs to allow read/passthru to the guest.
2484	*
2485	* The name of this function is slightly confusing. This function does NOT
2486	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
2487	* common prefix for functions dealing with "lazy restoration" of the shared
2488	* MSRs.
2489	*
2490	* @param pVCpu The cross context virtual CPU structure.
2491	*
2492	* @remarks No-long-jump zone!!!
2493	*/
2494	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
2495	{
2496	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2497	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2498
2499	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2500	#if HC_ARCH_BITS == 64
2501	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2502	{
2503	/*
2504	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
2505	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
2506	* we can skip a few MSR writes.
2507	*
2508	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
2509	* guest MSR values in the guest-CPU context might be different to what's currently
2510	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
2511	* CPU, see @bugref{8728}.
2512	*/
2513	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2514	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2515	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
2516	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
2517	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
2518	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
2519	{
2520	#ifdef VBOX_STRICT
2521	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
2522	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
2523	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
2524	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
2525	#endif
2526	}
2527	else
2528	{
2529	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
2530	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2531	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2532	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2533	}
2534	}
2535	#endif
2536	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2537	}
2538
2539
2540	/**
2541	* Performs lazy restoration of the set of host MSRs if they were previously
2542	* loaded with guest MSR values.
2543	*
2544	* @param pVCpu The cross context virtual CPU structure.
2545	*
2546	* @remarks No-long-jump zone!!!
2547	* @remarks The guest MSRs should have been saved back into the guest-CPU
2548	* context by hmR0VmxImportGuestState()!!!
2549	*/
2550	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
2551	{
2552	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2553	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2554
2555	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2556	{
2557	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2558	#if HC_ARCH_BITS == 64
2559	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2560	{
2561	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2562	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2563	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2564	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2565	}
2566	#endif
2567	}
2568	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2569	}
2570
2571
2572	/**
2573	* Verifies that our cached values of the VMCS fields are all consistent with
2574	* what's actually present in the VMCS.
2575	*
2576	* @returns VBox status code.
2577	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2578	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2579	* VMCS content. HMCPU error-field is
2580	* updated, see VMX_VCI_XXX.
2581	* @param pVCpu The cross context virtual CPU structure.
2582	* @param pVmcsInfo The VMCS info. object.
2583	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2584	*/
2585	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2586	{
2587	const char * const pcszVmcs = fIsNstGstVmcs ? "Nested-guest VMCS" : "VMCS";
2588
2589	uint32_t u32Val;
2590	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2591	AssertRCReturn(rc, rc);
2592	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2593	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32EntryCtls, u32Val),
2594	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2595	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2596
2597	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2598	AssertRCReturn(rc, rc);
2599	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2600	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ExitCtls, u32Val),
2601	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2602	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2603
2604	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2605	AssertRCReturn(rc, rc);
2606	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2607	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32PinCtls, u32Val),
2608	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2609	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2610
2611	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2612	AssertRCReturn(rc, rc);
2613	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2614	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls, u32Val),
2615	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2616	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2617
2618	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2619	{
2620	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2621	AssertRCReturn(rc, rc);
2622	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2623	("%s controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls2, u32Val),
2624	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2625	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2626	}
2627
2628	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2629	AssertRCReturn(rc, rc);
2630	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2631	("%s exception bitmap mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32XcptBitmap, u32Val),
2632	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2633	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2634
2635	uint64_t u64Val;
2636	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2637	AssertRCReturn(rc, rc);
2638	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2639	("%s TSC offset mismatch: Cache=%#RX64 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64TscOffset, u64Val),
2640	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2641	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2642
2643	NOREF(pcszVmcs);
2644	return VINF_SUCCESS;
2645	}
2646
2647
2648	#ifdef VBOX_STRICT
2649	/**
2650	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2651	*
2652	* @param pVCpu The cross context virtual CPU structure.
2653	* @param pVmcsInfo The VMCS info. object.
2654	*/
2655	static void hmR0VmxCheckHostEferMsr(PCVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2656	{
2657	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2658
2659	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2660	{
2661	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2662	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2663	uint64_t uVmcsEferMsrVmcs;
2664	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2665	AssertRC(rc);
2666
2667	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2668	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2669	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2670	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2671	}
2672	}
2673
2674
2675	/**
2676	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2677	* VMCS are correct.
2678	*
2679	* @param pVCpu The cross context virtual CPU structure.
2680	* @param pVmcsInfo The VMCS info. object.
2681	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2682	*/
2683	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2684	{
2685	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2686
2687	/* Read the various MSR-area counts from the VMCS. */
2688	uint32_t cEntryLoadMsrs;
2689	uint32_t cExitStoreMsrs;
2690	uint32_t cExitLoadMsrs;
2691	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cEntryLoadMsrs); AssertRC(rc);
2692	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cExitStoreMsrs); AssertRC(rc);
2693	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cExitLoadMsrs); AssertRC(rc);
2694
2695	/* Verify all the MSR counts are the same. */
2696	Assert(cEntryLoadMsrs == cExitStoreMsrs);
2697	Assert(cExitStoreMsrs == cExitLoadMsrs);
2698	uint32_t const cMsrs = cExitLoadMsrs;
2699
2700	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2701	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2702
2703	/* Verify the MSR counts are within the allocated page size. */
2704	Assert(sizeof(VMXAUTOMSR) * cMsrs <= X86_PAGE_4K_SIZE);
2705
2706	/* Verify the relevant contents of the MSR areas match. */
2707	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2708	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2709	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2710	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2711	for (uint32_t i = 0; i < cMsrs; i++)
2712	{
2713	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2714	if (fSeparateExitMsrStorePage)
2715	{
2716	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2717	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2718	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2719	}
2720
2721	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2722	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2723	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2724
2725	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2726	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2727	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2728	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2729
2730	/* Verify that cached host EFER MSR matches what's loaded the CPU. */
2731	bool const fIsEferMsr = RT_BOOL(pHostMsrLoad->u32Msr == MSR_K6_EFER);
2732	if (fIsEferMsr)
2733	{
2734	AssertMsgReturnVoid(u64Msr == pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer,
2735	("Cached=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2736	pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer, u64Msr, cMsrs));
2737	}
2738
2739	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2740	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2741	{
2742	uint32_t const fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2743	if (fIsEferMsr)
2744	{
2745	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2746	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2747	}
2748	else
2749	{
2750	if (!fIsNstGstVmcs)
2751	{
2752	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_ALLOW_RD_WR) == VMXMSRPM_ALLOW_RD_WR,
2753	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2754	}
2755	else
2756	{
2757	/*
2758	* A nested-guest VMCS must -also- allow read/write passthrough for the MSR for us to
2759	* execute a nested-guest with MSR passthrough.
2760	*
2761	* Check if the nested-guest MSR bitmap allows passthrough, and if so, assert that we
2762	* allow passthrough too.
2763	*/
2764	void const *pvMsrBitmapNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
2765	Assert(pvMsrBitmapNstGst);
2766	uint32_t const fMsrpmNstGst = CPUMGetVmxMsrPermission(pvMsrBitmapNstGst, pGuestMsrLoad->u32Msr);
2767	AssertMsgReturnVoid(fMsrpm == fMsrpmNstGst,
2768	("u32Msr=%#RX32 cMsrs=%u Permission mismatch fMsrpm=%#x fMsrpmNstGst=%#x!\n",
2769	pGuestMsrLoad->u32Msr, cMsrs, fMsrpm, fMsrpmNstGst));
2770	}
2771	}
2772	}
2773
2774	/* Move to the next MSR. */
2775	pHostMsrLoad++;
2776	pGuestMsrLoad++;
2777	pGuestMsrStore++;
2778	}
2779	}
2780	#endif /* VBOX_STRICT */
2781
2782
2783	/**
2784	* Flushes the TLB using EPT.
2785	*
2786	* @returns VBox status code.
2787	* @param pVCpu The cross context virtual CPU structure of the calling
2788	* EMT. Can be NULL depending on @a enmTlbFlush.
2789	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2790	* enmTlbFlush.
2791	* @param enmTlbFlush Type of flush.
2792	*
2793	* @remarks Caller is responsible for making sure this function is called only
2794	* when NestedPaging is supported and providing @a enmTlbFlush that is
2795	* supported by the CPU.
2796	* @remarks Can be called with interrupts disabled.
2797	*/
2798	static void hmR0VmxFlushEpt(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2799	{
2800	uint64_t au64Descriptor[2];
2801	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2802	au64Descriptor[0] = 0;
2803	else
2804	{
2805	Assert(pVCpu);
2806	Assert(pVmcsInfo);
2807	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2808	}
2809	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2810
2811	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2812	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2813
2814	if ( RT_SUCCESS(rc)
2815	&& pVCpu)
2816	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2817	}
2818
2819
2820	/**
2821	* Flushes the TLB using VPID.
2822	*
2823	* @returns VBox status code.
2824	* @param pVCpu The cross context virtual CPU structure of the calling
2825	* EMT. Can be NULL depending on @a enmTlbFlush.
2826	* @param enmTlbFlush Type of flush.
2827	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2828	* on @a enmTlbFlush).
2829	*
2830	* @remarks Can be called with interrupts disabled.
2831	*/
2832	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2833	{
2834	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2835
2836	uint64_t au64Descriptor[2];
2837	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2838	{
2839	au64Descriptor[0] = 0;
2840	au64Descriptor[1] = 0;
2841	}
2842	else
2843	{
2844	AssertPtr(pVCpu);
2845	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2846	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2847	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2848	au64Descriptor[1] = GCPtr;
2849	}
2850
2851	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2852	AssertMsg(rc == VINF_SUCCESS,
2853	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2854
2855	if ( RT_SUCCESS(rc)
2856	&& pVCpu)
2857	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2858	NOREF(rc);
2859	}
2860
2861
2862	/**
2863	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2864	* otherwise there is nothing really to invalidate.
2865	*
2866	* @returns VBox status code.
2867	* @param pVCpu The cross context virtual CPU structure.
2868	* @param GCVirt Guest virtual address of the page to invalidate.
2869	*/
2870	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
2871	{
2872	AssertPtr(pVCpu);
2873	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2874
2875	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2876	{
2877	/*
2878	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2879	* the EPT case. See @bugref{6043} and @bugref{6177}.
2880	*
2881	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2882	* as this function maybe called in a loop with individual addresses.
2883	*/
2884	PVM pVM = pVCpu->CTX_SUFF(pVM);
2885	if (pVM->hm.s.vmx.fVpid)
2886	{
2887	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2888	if (fVpidFlush)
2889	{
2890	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2891	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2892	}
2893	else
2894	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2895	}
2896	else if (pVM->hm.s.fNestedPaging)
2897	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2898	}
2899
2900	return VINF_SUCCESS;
2901	}
2902
2903
2904	/**
2905	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2906	* case where neither EPT nor VPID is supported by the CPU.
2907	*
2908	* @param pHostCpu The HM physical-CPU structure.
2909	* @param pVCpu The cross context virtual CPU structure.
2910	*
2911	* @remarks Called with interrupts disabled.
2912	*/
2913	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2914	{
2915	AssertPtr(pVCpu);
2916	AssertPtr(pHostCpu);
2917
2918	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2919
2920	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2921	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2922	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2923	pVCpu->hm.s.fForceTLBFlush = false;
2924	return;
2925	}
2926
2927
2928	/**
2929	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2930	*
2931	* @param pHostCpu The HM physical-CPU structure.
2932	* @param pVCpu The cross context virtual CPU structure.
2933	* @param pVmcsInfo The VMCS info. object.
2934	*
2935	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2936	* nomenclature. The reason is, to avoid confusion in compare statements
2937	* since the host-CPU copies are named "ASID".
2938	*
2939	* @remarks Called with interrupts disabled.
2940	*/
2941	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2942	{
2943	#ifdef VBOX_WITH_STATISTICS
2944	bool fTlbFlushed = false;
2945	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2946	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2947	if (!fTlbFlushed) \
2948	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2949	} while (0)
2950	#else
2951	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2952	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2953	#endif
2954
2955	AssertPtr(pVCpu);
2956	AssertPtr(pHostCpu);
2957	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2958
2959	PVM pVM = pVCpu->CTX_SUFF(pVM);
2960	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2961	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2962	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2963
2964	/*
2965	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
2966	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2967	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2968	* cannot reuse the current ASID anymore.
2969	*/
2970	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2971	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2972	{
2973	++pHostCpu->uCurrentAsid;
2974	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2975	{
2976	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2977	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2978	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2979	}
2980
2981	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2982	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2983	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2984
2985	/*
2986	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2987	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2988	*/
2989	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2990	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2991	HMVMX_SET_TAGGED_TLB_FLUSHED();
2992	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2993	}
2994	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2995	{
2996	/*
2997	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2998	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2999	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
3000	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
3001	* mappings, see @bugref{6568}.
3002	*
3003	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
3004	*/
3005	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
3006	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3007	HMVMX_SET_TAGGED_TLB_FLUSHED();
3008	}
3009	else if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
3010	{
3011	/*
3012	* The nested-guest specifies its own guest-physical address to use as the APIC-access
3013	* address which requires flushing the TLB of EPT cached structures.
3014	*
3015	* See Intel spec. 28.3.3.4 "Guidelines for Use of the INVEPT Instruction".
3016	*/
3017	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
3018	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
3019	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
3020	HMVMX_SET_TAGGED_TLB_FLUSHED();
3021	}
3022
3023
3024	pVCpu->hm.s.fForceTLBFlush = false;
3025	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
3026
3027	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
3028	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
3029	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
3030	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
3031	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
3032	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
3033	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
3034	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
3035	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
3036
3037	/* Update VMCS with the VPID. */
3038	int rc = VMXWriteVmcs16(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
3039	AssertRC(rc);
3040
3041	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
3042	}
3043
3044
3045	/**
3046	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
3047	*
3048	* @param pHostCpu The HM physical-CPU structure.
3049	* @param pVCpu The cross context virtual CPU structure.
3050	* @param pVmcsInfo The VMCS info. object.
3051	*
3052	* @remarks Called with interrupts disabled.
3053	*/
3054	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
3055	{
3056	AssertPtr(pVCpu);
3057	AssertPtr(pHostCpu);
3058	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3059	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
3060	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
3061
3062	/*
3063	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
3064	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
3065	*/
3066	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3067	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3068	{
3069	pVCpu->hm.s.fForceTLBFlush = true;
3070	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3071	}
3072
3073	/* Check for explicit TLB flushes. */
3074	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3075	{
3076	pVCpu->hm.s.fForceTLBFlush = true;
3077	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3078	}
3079
3080	/* Check for TLB flushes while switching to/from a nested-guest. */
3081	if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
3082	{
3083	pVCpu->hm.s.fForceTLBFlush = true;
3084	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
3085	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
3086	}
3087
3088	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3089	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3090
3091	if (pVCpu->hm.s.fForceTLBFlush)
3092	{
3093	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
3094	pVCpu->hm.s.fForceTLBFlush = false;
3095	}
3096	}
3097
3098
3099	/**
3100	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
3101	*
3102	* @param pHostCpu The HM physical-CPU structure.
3103	* @param pVCpu The cross context virtual CPU structure.
3104	*
3105	* @remarks Called with interrupts disabled.
3106	*/
3107	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
3108	{
3109	AssertPtr(pVCpu);
3110	AssertPtr(pHostCpu);
3111	Assert(pHostCpu->idCpu != NIL_RTCPUID);
3112	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
3113	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
3114
3115	/*
3116	* Force a TLB flush for the first world switch if the current CPU differs from the one we
3117	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
3118	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
3119	* cannot reuse the current ASID anymore.
3120	*/
3121	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
3122	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
3123	{
3124	pVCpu->hm.s.fForceTLBFlush = true;
3125	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
3126	}
3127
3128	/* Check for explicit TLB flushes. */
3129	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
3130	{
3131	/*
3132	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
3133	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
3134	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
3135	* include fExplicitFlush's too) - an obscure corner case.
3136	*/
3137	pVCpu->hm.s.fForceTLBFlush = true;
3138	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
3139	}
3140
3141	/* Check for TLB flushes while switching to/from a nested-guest. */
3142	if (pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb)
3143	{
3144	pVCpu->hm.s.fForceTLBFlush = true;
3145	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = false;
3146	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbNstGst);
3147	}
3148
3149	PVM pVM = pVCpu->CTX_SUFF(pVM);
3150	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
3151	if (pVCpu->hm.s.fForceTLBFlush)
3152	{
3153	++pHostCpu->uCurrentAsid;
3154	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
3155	{
3156	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
3157	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
3158	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
3159	}
3160
3161	pVCpu->hm.s.fForceTLBFlush = false;
3162	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
3163	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
3164	if (pHostCpu->fFlushAsidBeforeUse)
3165	{
3166	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
3167	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
3168	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
3169	{
3170	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
3171	pHostCpu->fFlushAsidBeforeUse = false;
3172	}
3173	else
3174	{
3175	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
3176	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
3177	}
3178	}
3179	}
3180
3181	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
3182	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
3183	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
3184	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
3185	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
3186	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
3187	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
3188
3189	int rc = VMXWriteVmcs16(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
3190	AssertRC(rc);
3191	}
3192
3193
3194	/**
3195	* Flushes the guest TLB entry based on CPU capabilities.
3196	*
3197	* @param pHostCpu The HM physical-CPU structure.
3198	* @param pVCpu The cross context virtual CPU structure.
3199	* @param pVmcsInfo The VMCS info. object.
3200	*
3201	* @remarks Called with interrupts disabled.
3202	*/
3203	static void hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3204	{
3205	#ifdef HMVMX_ALWAYS_FLUSH_TLB
3206	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
3207	#endif
3208	PVM pVM = pVCpu->CTX_SUFF(pVM);
3209	switch (pVM->hm.s.vmx.enmTlbFlushType)
3210	{
3211	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
3212	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
3213	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
3214	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
3215	default:
3216	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
3217	break;
3218	}
3219	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
3220	}
3221
3222
3223	/**
3224	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
3225	* TLB entries from the host TLB before VM-entry.
3226	*
3227	* @returns VBox status code.
3228	* @param pVM The cross context VM structure.
3229	*/
3230	static int hmR0VmxSetupTaggedTlb(PVM pVM)
3231	{
3232	/*
3233	* Determine optimal flush type for nested paging.
3234	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
3235	* unrestricted guest execution (see hmR3InitFinalizeR0()).
3236	*/
3237	if (pVM->hm.s.fNestedPaging)
3238	{
3239	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
3240	{
3241	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
3242	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
3243	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3244	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
3245	else
3246	{
3247	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
3248	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3249	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
3250	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3251	}
3252
3253	/* Make sure the write-back cacheable memory type for EPT is supported. */
3254	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
3255	{
3256	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3257	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
3258	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3259	}
3260
3261	/* EPT requires a page-walk length of 4. */
3262	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
3263	{
3264	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3265	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
3266	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3267	}
3268	}
3269	else
3270	{
3271	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
3272	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
3273	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
3274	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3275	}
3276	}
3277
3278	/*
3279	* Determine optimal flush type for VPID.
3280	*/
3281	if (pVM->hm.s.vmx.fVpid)
3282	{
3283	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
3284	{
3285	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
3286	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
3287	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
3288	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
3289	else
3290	{
3291	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
3292	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
3293	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
3294	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
3295	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
3296	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3297	pVM->hm.s.vmx.fVpid = false;
3298	}
3299	}
3300	else
3301	{
3302	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
3303	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
3304	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
3305	pVM->hm.s.vmx.fVpid = false;
3306	}
3307	}
3308
3309	/*
3310	* Setup the handler for flushing tagged-TLBs.
3311	*/
3312	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
3313	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
3314	else if (pVM->hm.s.fNestedPaging)
3315	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
3316	else if (pVM->hm.s.vmx.fVpid)
3317	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
3318	else
3319	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
3320	return VINF_SUCCESS;
3321	}
3322
3323
3324	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3325	/**
3326	* Sets up the shadow VMCS fields arrays.
3327	*
3328	* This function builds arrays of VMCS fields to sync the shadow VMCS later while
3329	* executing the guest.
3330	*
3331	* @returns VBox status code.
3332	* @param pVM The cross context VM structure.
3333	*/
3334	static int hmR0VmxSetupShadowVmcsFieldsArrays(PVM pVM)
3335	{
3336	/*
3337	* Paranoia. Ensure we haven't exposed the VMWRITE-All VMX feature to the guest
3338	* when the host does not support it.
3339	*/
3340	bool const fGstVmwriteAll = pVM->cpum.ro.GuestFeatures.fVmxVmwriteAll;
3341	if ( !fGstVmwriteAll
3342	\|\| (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL))
3343	{ /* likely. */ }
3344	else
3345	{
3346	LogRelFunc(("VMX VMWRITE-All feature exposed to the guest but host CPU does not support it!\n"));
3347	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_GST_HOST_VMWRITE_ALL;
3348	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3349	}
3350
3351	uint32_t const cVmcsFields = RT_ELEMENTS(g_aVmcsFields);
3352	uint32_t cRwFields = 0;
3353	uint32_t cRoFields = 0;
3354	for (uint32_t i = 0; i < cVmcsFields; i++)
3355	{
3356	VMXVMCSFIELD VmcsField;
3357	VmcsField.u = g_aVmcsFields[i];
3358
3359	/*
3360	* We will be writing "FULL" (64-bit) fields while syncing the shadow VMCS.
3361	* Therefore, "HIGH" (32-bit portion of 64-bit) fields must not be included
3362	* in the shadow VMCS fields array as they would be redundant.
3363	*
3364	* If the VMCS field depends on a CPU feature that is not exposed to the guest,
3365	* we must not include it in the shadow VMCS fields array. Guests attempting to
3366	* VMREAD/VMWRITE such VMCS fields would cause a VM-exit and we shall emulate
3367	* the required behavior.
3368	*/
3369	if ( VmcsField.n.fAccessType == VMX_VMCSFIELD_ACCESS_FULL
3370	&& CPUMIsGuestVmxVmcsFieldValid(pVM, VmcsField.u))
3371	{
3372	/*
3373	* Read-only fields are placed in a separate array so that while syncing shadow
3374	* VMCS fields later (which is more performance critical) we can avoid branches.
3375	*
3376	* However, if the guest can write to all fields (including read-only fields),
3377	* we treat it a as read/write field. Otherwise, writing to these fields would
3378	* cause a VMWRITE instruction error while syncing the shadow VMCS .
3379	*/
3380	if ( fGstVmwriteAll
3381	\|\| !HMVmxIsVmcsFieldReadOnly(VmcsField.u))
3382	pVM->hm.s.vmx.paShadowVmcsFields[cRwFields++] = VmcsField.u;
3383	else
3384	pVM->hm.s.vmx.paShadowVmcsRoFields[cRoFields++] = VmcsField.u;
3385	}
3386	}
3387
3388	/* Update the counts. */
3389	pVM->hm.s.vmx.cShadowVmcsFields = cRwFields;
3390	pVM->hm.s.vmx.cShadowVmcsRoFields = cRoFields;
3391	return VINF_SUCCESS;
3392	}
3393
3394
3395	/**
3396	* Sets up the VMREAD and VMWRITE bitmaps.
3397	*
3398	* @param pVM The cross context VM structure.
3399	*/
3400	static void hmR0VmxSetupVmreadVmwriteBitmaps(PVM pVM)
3401	{
3402	/*
3403	* By default, ensure guest attempts to acceses to any VMCS fields cause VM-exits.
3404	*/
3405	uint32_t const cbBitmap = X86_PAGE_4K_SIZE;
3406	uint8_t pbVmreadBitmap = (uint8_t )pVM->hm.s.vmx.pvVmreadBitmap;
3407	uint8_t pbVmwriteBitmap = (uint8_t )pVM->hm.s.vmx.pvVmwriteBitmap;
3408	ASMMemFill32(pbVmreadBitmap, cbBitmap, UINT32_C(0xffffffff));
3409	ASMMemFill32(pbVmwriteBitmap, cbBitmap, UINT32_C(0xffffffff));
3410
3411	/*
3412	* Skip intercepting VMREAD/VMWRITE to guest read/write fields in the
3413	* VMREAD and VMWRITE bitmaps.
3414	*/
3415	{
3416	uint32_t const *paShadowVmcsFields = pVM->hm.s.vmx.paShadowVmcsFields;
3417	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
3418	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
3419	{
3420	uint32_t const uVmcsField = paShadowVmcsFields[i];
3421	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
3422	Assert(uVmcsField >> 3 < cbBitmap);
3423	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
3424	ASMBitClear(pbVmwriteBitmap + (uVmcsField >> 3), uVmcsField & 7);
3425	}
3426	}
3427
3428	/*
3429	* Skip intercepting VMREAD for guest read-only fields in the VMREAD bitmap
3430	* if the host supports VMWRITE to all supported VMCS fields.
3431	*/
3432	if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
3433	{
3434	uint32_t const *paShadowVmcsRoFields = pVM->hm.s.vmx.paShadowVmcsRoFields;
3435	uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
3436	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
3437	{
3438	uint32_t const uVmcsField = paShadowVmcsRoFields[i];
3439	Assert(!(uVmcsField & VMX_VMCSFIELD_RSVD_MASK));
3440	Assert(uVmcsField >> 3 < cbBitmap);
3441	ASMBitClear(pbVmreadBitmap + (uVmcsField >> 3), uVmcsField & 7);
3442	}
3443	}
3444	}
3445	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
3446
3447
3448	/**
3449	* Sets up the virtual-APIC page address for the VMCS.
3450	*
3451	* @returns VBox status code.
3452	* @param pVCpu The cross context virtual CPU structure.
3453	* @param pVmcsInfo The VMCS info. object.
3454	*/
3455	DECLINLINE(int) hmR0VmxSetupVmcsVirtApicAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
3456	{
3457	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3458	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
3459	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
3460	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
3461	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
3462	}
3463
3464
3465	/**
3466	* Sets up the MSR-bitmap address for the VMCS.
3467	*
3468	* @returns VBox status code.
3469	* @param pVCpu The cross context virtual CPU structure.
3470	* @param pVmcsInfo The VMCS info. object.
3471	*/
3472	DECLINLINE(int) hmR0VmxSetupVmcsMsrBitmapAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
3473	{
3474	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3475	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
3476	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
3477	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3478	return VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
3479	}
3480
3481
3482	/**
3483	* Sets up the APIC-access page address for the VMCS.
3484	*
3485	* @returns VBox status code.
3486	* @param pVCpu The cross context virtual CPU structure.
3487	*/
3488	DECLINLINE(int) hmR0VmxSetupVmcsApicAccessAddr(PVMCPU pVCpu)
3489	{
3490	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
3491	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
3492	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
3493	return VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
3494	}
3495
3496
3497	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3498	/**
3499	* Sets up the VMREAD bitmap address for the VMCS.
3500	*
3501	* @returns VBox status code.
3502	* @param pVCpu The cross context virtual CPU structure.
3503	*/
3504	DECLINLINE(int) hmR0VmxSetupVmcsVmreadBitmapAddr(PVMCPU pVCpu)
3505	{
3506	RTHCPHYS const HCPhysVmreadBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmreadBitmap;
3507	Assert(HCPhysVmreadBitmap != NIL_RTHCPHYS);
3508	Assert(!(HCPhysVmreadBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3509	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL, HCPhysVmreadBitmap);
3510	}
3511
3512
3513	/**
3514	* Sets up the VMWRITE bitmap address for the VMCS.
3515	*
3516	* @returns VBox status code.
3517	* @param pVCpu The cross context virtual CPU structure.
3518	*/
3519	DECLINLINE(int) hmR0VmxSetupVmcsVmwriteBitmapAddr(PVMCPU pVCpu)
3520	{
3521	RTHCPHYS const HCPhysVmwriteBitmap = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysVmwriteBitmap;
3522	Assert(HCPhysVmwriteBitmap != NIL_RTHCPHYS);
3523	Assert(!(HCPhysVmwriteBitmap & 0xfff)); /* Bits 11:0 MBZ. */
3524	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL, HCPhysVmwriteBitmap);
3525	}
3526	#endif
3527
3528
3529	/**
3530	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
3531	* in the VMCS.
3532	*
3533	* @returns VBox status code.
3534	* @param pVCpu The cross context virtual CPU structure.
3535	* @param pVmcsInfo The VMCS info. object.
3536	*/
3537	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3538	{
3539	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
3540
3541	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
3542	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
3543	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3544
3545	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
3546	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
3547	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
3548
3549	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
3550	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
3551	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
3552
3553	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad);
3554	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore);
3555	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad);
3556	AssertRCReturn(rc, rc);
3557	return VINF_SUCCESS;
3558	}
3559
3560
3561	/**
3562	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
3563	*
3564	* @param pVCpu The cross context virtual CPU structure.
3565	* @param pVmcsInfo The VMCS info. object.
3566	*/
3567	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3568	{
3569	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
3570
3571	/*
3572	* The guest can access the following MSRs (read, write) without causing
3573	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
3574	*/
3575	PVM pVM = pVCpu->CTX_SUFF(pVM);
3576	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD_WR);
3577	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD_WR);
3578	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD_WR);
3579	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3580	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD_WR);
3581
3582	/*
3583	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
3584	* associated with then. We never need to intercept access (writes need to be
3585	* executed without causing a VM-exit, reads will #GP fault anyway).
3586	*
3587	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
3588	* read/write them. We swap the the guest/host MSR value using the
3589	* auto-load/store MSR area.
3590	*/
3591	if (pVM->cpum.ro.GuestFeatures.fIbpb)
3592	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD_WR);
3593	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
3594	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD_WR);
3595	if (pVM->cpum.ro.GuestFeatures.fIbrs)
3596	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD_WR);
3597
3598	#if HC_ARCH_BITS == 64
3599	/*
3600	* Allow full read/write access for the following MSRs (mandatory for VT-x)
3601	* required for 64-bit guests.
3602	*/
3603	if (pVM->hm.s.fAllow64BitGuests)
3604	{
3605	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD_WR);
3606	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K6_STAR, VMXMSRPM_ALLOW_RD_WR);
3607	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD_WR);
3608	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, false, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD_WR);
3609	}
3610	#endif
3611
3612	/*
3613	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
3614	*/
3615	#ifdef VBOX_STRICT
3616	Assert(pVmcsInfo->pvMsrBitmap);
3617	uint32_t const fMsrpmEfer = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K6_EFER);
3618	Assert(fMsrpmEfer == VMXMSRPM_EXIT_RD_WR);
3619	#endif
3620	}
3621
3622
3623	/**
3624	* Sets up pin-based VM-execution controls in the VMCS.
3625	*
3626	* @returns VBox status code.
3627	* @param pVCpu The cross context virtual CPU structure.
3628	* @param pVmcsInfo The VMCS info. object.
3629	*/
3630	static int hmR0VmxSetupVmcsPinCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3631	{
3632	PVM pVM = pVCpu->CTX_SUFF(pVM);
3633	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
3634	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
3635
3636	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
3637	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
3638
3639	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
3640	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
3641
3642	/* Enable the VMX-preemption timer. */
3643	if (pVM->hm.s.vmx.fUsePreemptTimer)
3644	{
3645	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
3646	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
3647	}
3648
3649	#if 0
3650	/* Enable posted-interrupt processing. */
3651	if (pVM->hm.s.fPostedIntrs)
3652	{
3653	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
3654	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
3655	fVal \|= VMX_PIN_CTLS_POSTED_INT;
3656	}
3657	#endif
3658
3659	if ((fVal & fZap) != fVal)
3660	{
3661	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3662	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
3663	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
3664	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3665	}
3666
3667	/* Commit it to the VMCS and update our cache. */
3668	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
3669	AssertRCReturn(rc, rc);
3670	pVmcsInfo->u32PinCtls = fVal;
3671
3672	return VINF_SUCCESS;
3673	}
3674
3675
3676	/**
3677	* Sets up secondary processor-based VM-execution controls in the VMCS.
3678	*
3679	* @returns VBox status code.
3680	* @param pVCpu The cross context virtual CPU structure.
3681	* @param pVmcsInfo The VMCS info. object.
3682	*/
3683	static int hmR0VmxSetupVmcsProcCtls2(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3684	{
3685	PVM pVM = pVCpu->CTX_SUFF(pVM);
3686	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
3687	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3688
3689	/* WBINVD causes a VM-exit. */
3690	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
3691	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
3692
3693	/* Enable EPT (aka nested-paging). */
3694	if (pVM->hm.s.fNestedPaging)
3695	fVal \|= VMX_PROC_CTLS2_EPT;
3696
3697	/* Enable the INVPCID instruction if we expose it to the guest and is supported
3698	by the hardware. Without this, guest executing INVPCID would cause a #UD. */
3699	if ( pVM->cpum.ro.GuestFeatures.fInvpcid
3700	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID))
3701	fVal \|= VMX_PROC_CTLS2_INVPCID;
3702
3703	/* Enable VPID. */
3704	if (pVM->hm.s.vmx.fVpid)
3705	fVal \|= VMX_PROC_CTLS2_VPID;
3706
3707	/* Enable unrestricted guest execution. */
3708	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3709	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
3710
3711	#if 0
3712	if (pVM->hm.s.fVirtApicRegs)
3713	{
3714	/* Enable APIC-register virtualization. */
3715	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
3716	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
3717
3718	/* Enable virtual-interrupt delivery. */
3719	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
3720	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
3721	}
3722	#endif
3723
3724	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is
3725	where the TPR shadow resides. */
3726	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3727	* done dynamically. */
3728	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3729	{
3730	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3731	int rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3732	AssertRCReturn(rc, rc);
3733	}
3734
3735	/* Enable the RDTSCP instruction if we expose it to the guest and is supported
3736	by the hardware. Without this, guest executing RDTSCP would cause a #UD. */
3737	if ( pVM->cpum.ro.GuestFeatures.fRdTscP
3738	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP))
3739	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3740
3741	/* Enable Pause-Loop exiting. */
3742	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
3743	&& pVM->hm.s.vmx.cPleGapTicks
3744	&& pVM->hm.s.vmx.cPleWindowTicks)
3745	{
3746	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3747
3748	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
3749	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
3750	AssertRCReturn(rc, rc);
3751	}
3752
3753	if ((fVal & fZap) != fVal)
3754	{
3755	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3756	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3757	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3758	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3759	}
3760
3761	/* Commit it to the VMCS and update our cache. */
3762	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3763	AssertRCReturn(rc, rc);
3764	pVmcsInfo->u32ProcCtls2 = fVal;
3765
3766	return VINF_SUCCESS;
3767	}
3768
3769
3770	/**
3771	* Sets up processor-based VM-execution controls in the VMCS.
3772	*
3773	* @returns VBox status code.
3774	* @param pVCpu The cross context virtual CPU structure.
3775	* @param pVmcsInfo The VMCS info. object.
3776	*/
3777	static int hmR0VmxSetupVmcsProcCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3778	{
3779	PVM pVM = pVCpu->CTX_SUFF(pVM);
3780
3781	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3782	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3783
3784	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3785	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3786	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3787	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3788	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3789	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3790	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3791
3792	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3793	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3794	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3795	{
3796	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3797	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3798	}
3799
3800	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3801	if (!pVM->hm.s.fNestedPaging)
3802	{
3803	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3804	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3805	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3806	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3807	}
3808
3809	/* Use TPR shadowing if supported by the CPU. */
3810	if ( PDMHasApic(pVM)
3811	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3812	{
3813	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3814	/* CR8 writes cause a VM-exit based on TPR threshold. */
3815	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3816	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3817	int rc = hmR0VmxSetupVmcsVirtApicAddr(pVCpu, pVmcsInfo);
3818	AssertRCReturn(rc, rc);
3819	}
3820	else
3821	{
3822	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3823	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3824	if (pVM->hm.s.fAllow64BitGuests)
3825	{
3826	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3827	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3828	}
3829	}
3830
3831	/* Use MSR-bitmaps if supported by the CPU. */
3832	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3833	{
3834	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3835	int rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3836	AssertRCReturn(rc, rc);
3837	}
3838
3839	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3840	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3841	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3842
3843	if ((fVal & fZap) != fVal)
3844	{
3845	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3846	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3847	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3848	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3849	}
3850
3851	/* Commit it to the VMCS and update our cache. */
3852	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3853	AssertRCReturn(rc, rc);
3854	pVmcsInfo->u32ProcCtls = fVal;
3855
3856	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3857	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3858	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo);
3859
3860	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3861	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3862	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3863
3864	/* Sanity check, should not really happen. */
3865	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3866	{ /* likely */ }
3867	else
3868	{
3869	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3870	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3871	}
3872
3873	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3874	return VINF_SUCCESS;
3875	}
3876
3877
3878	/**
3879	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3880	* Processor-based VM-execution) control fields in the VMCS.
3881	*
3882	* @returns VBox status code.
3883	* @param pVCpu The cross context virtual CPU structure.
3884	* @param pVmcsInfo The VMCS info. object.
3885	*/
3886	static int hmR0VmxSetupVmcsMiscCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3887	{
3888	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3889	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
3890	{
3891	int rc = hmR0VmxSetupVmcsVmreadBitmapAddr(pVCpu);
3892	rc \|= hmR0VmxSetupVmcsVmwriteBitmapAddr(pVCpu);
3893	if (RT_SUCCESS(rc))
3894	{ /* likely */ }
3895	else
3896	{
3897	LogRelFunc(("Failed to setup VMREAD/VMWRITE bitmap addresses. rc=%Rrc\n", rc));
3898	return rc;
3899	}
3900	}
3901	#endif
3902
3903	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
3904	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
3905	if (RT_SUCCESS(rc))
3906	{
3907	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3908	if (RT_SUCCESS(rc))
3909	{
3910	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3911	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3912	rc = VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
3913	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
3914	if (RT_SUCCESS(rc))
3915	{
3916	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3917	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3918	return VINF_SUCCESS;
3919	}
3920	LogRelFunc(("Failed to initialize VMCS CR0/CR4 guest/host mask. rc=%Rrc\n", rc));
3921	}
3922	else
3923	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3924	}
3925	else
3926	LogRelFunc(("Failed to initialize VMCS link pointer. rc=%Rrc\n", rc));
3927	return rc;
3928	}
3929
3930
3931	/**
3932	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3933	*
3934	* We shall setup those exception intercepts that don't change during the
3935	* lifetime of the VM here. The rest are done dynamically while loading the
3936	* guest state.
3937	*
3938	* @returns VBox status code.
3939	* @param pVCpu The cross context virtual CPU structure.
3940	* @param pVmcsInfo The VMCS info. object.
3941	*/
3942	static int hmR0VmxSetupVmcsXcptBitmap(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3943	{
3944	/*
3945	* The following exceptions are always intercepted:
3946	*
3947	* #AC - To prevent the guest from hanging the CPU.
3948	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3949	* recursive #DBs can cause a CPU hang.
3950	* #PF - To sync our shadow page tables when nested-paging is not used.
3951	*/
3952	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3953	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3954	\| RT_BIT(X86_XCPT_DB)
3955	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3956
3957	/* Commit it to the VMCS. */
3958	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3959	AssertRCReturn(rc, rc);
3960
3961	/* Update our cache of the exception bitmap. */
3962	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3963	return VINF_SUCCESS;
3964	}
3965
3966
3967	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3968	/**
3969	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3970	*
3971	* @returns VBox status code.
3972	* @param pVCpu The cross context virtual CPU structure.
3973	* @param pVmcsInfo The VMCS info. object.
3974	*/
3975	static int hmR0VmxSetupVmcsCtlsNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3976	{
3977	PVM pVM = pVCpu->CTX_SUFF(pVM);
3978	Assert(pVmcsInfo->u64VmcsLinkPtr == NIL_RTHCPHYS);
3979	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
3980	if (RT_SUCCESS(rc))
3981	{
3982	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3983	if (RT_SUCCESS(rc))
3984	{
3985	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3986	rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3987	if (RT_SUCCESS(rc))
3988	return VINF_SUCCESS;
3989	LogRelFunc(("Failed to set up the MSR-bitmap address in the nested-guest VMCS. rc=%Rrc\n", rc));
3990	}
3991	else
3992	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3993	}
3994	else
3995	LogRelFunc(("Failed to set up the auto-load/store MSR addresses in the nested-guest VMCS. rc=%Rrc\n", rc));
3996
3997	return rc;
3998	}
3999	#endif
4000
4001
4002	/**
4003	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
4004	* VMX.
4005	*
4006	* @returns VBox status code.
4007	* @param pVCpu The cross context virtual CPU structure.
4008	* @param pVmcsInfo The VMCS info. object.
4009	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
4010	*/
4011	static int hmR0VmxSetupVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
4012	{
4013	Assert(pVmcsInfo->pvVmcs);
4014	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4015
4016	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
4017	PVM pVM = pVCpu->CTX_SUFF(pVM);
4018	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
4019	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
4020
4021	LogFlowFunc(("\n"));
4022
4023	/*
4024	* Initialize the VMCS using VMCLEAR before loading the VMCS.
4025	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
4026	*/
4027	int rc = hmR0VmxClearVmcs(pVmcsInfo);
4028	if (RT_SUCCESS(rc))
4029	{
4030	rc = hmR0VmxLoadVmcs(pVmcsInfo);
4031	if (RT_SUCCESS(rc))
4032	{
4033	if (!fIsNstGstVmcs)
4034	{
4035	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
4036	if (RT_SUCCESS(rc))
4037	{
4038	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
4039	if (RT_SUCCESS(rc))
4040	{
4041	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
4042	if (RT_SUCCESS(rc))
4043	{
4044	rc = hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
4045	if (RT_SUCCESS(rc))
4046	{
4047	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4048	/*
4049	* If a shadow VMCS is allocated for the VMCS info. object, initialize the
4050	* VMCS revision ID and shadow VMCS indicator bit. Also, clear the VMCS
4051	* making it fit for use when VMCS shadowing is later enabled.
4052	*/
4053	if (pVmcsInfo->pvShadowVmcs)
4054	{
4055	VMXVMCSREVID VmcsRevId;
4056	VmcsRevId.u = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
4057	VmcsRevId.n.fIsShadowVmcs = 1;
4058	(uint32_t )pVmcsInfo->pvShadowVmcs = VmcsRevId.u;
4059	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
4060	if (RT_SUCCESS(rc))
4061	{ /* likely */ }
4062	else
4063	LogRelFunc(("Failed to initialize shadow VMCS. rc=%Rrc\n", rc));
4064	}
4065	#endif
4066	}
4067	else
4068	LogRelFunc(("Failed to initialize exception bitmap. rc=%Rrc\n", rc));
4069	}
4070	else
4071	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
4072	}
4073	else
4074	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
4075	}
4076	else
4077	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
4078	}
4079	else
4080	{
4081	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4082	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
4083	if (RT_SUCCESS(rc))
4084	{ /* likely */ }
4085	else
4086	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
4087	#else
4088	AssertFailed();
4089	#endif
4090	}
4091	}
4092	else
4093	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
4094	}
4095	else
4096	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
4097
4098	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
4099	if (RT_SUCCESS(rc))
4100	{
4101	rc = hmR0VmxClearVmcs(pVmcsInfo);
4102	if (RT_SUCCESS(rc))
4103	{ /* likely */ }
4104	else
4105	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
4106	}
4107
4108	/*
4109	* Update the last-error record both for failures and success, so we
4110	* can propagate the status code back to ring-3 for diagnostics.
4111	*/
4112	hmR0VmxUpdateErrorRecord(pVCpu, rc);
4113	NOREF(pszVmcs);
4114	return rc;
4115	}
4116
4117
4118	/**
4119	* Does global VT-x initialization (called during module initialization).
4120	*
4121	* @returns VBox status code.
4122	*/
4123	VMMR0DECL(int) VMXR0GlobalInit(void)
4124	{
4125	#ifdef HMVMX_USE_FUNCTION_TABLE
4126	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
4127	# ifdef VBOX_STRICT
4128	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
4129	Assert(g_apfnVMExitHandlers[i]);
4130	# endif
4131	#endif
4132	return VINF_SUCCESS;
4133	}
4134
4135
4136	/**
4137	* Does global VT-x termination (called during module termination).
4138	*/
4139	VMMR0DECL(void) VMXR0GlobalTerm()
4140	{
4141	/* Nothing to do currently. */
4142	}
4143
4144
4145	/**
4146	* Sets up and activates VT-x on the current CPU.
4147	*
4148	* @returns VBox status code.
4149	* @param pHostCpu The HM physical-CPU structure.
4150	* @param pVM The cross context VM structure. Can be
4151	* NULL after a host resume operation.
4152	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
4153	* fEnabledByHost is @c true).
4154	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
4155	* @a fEnabledByHost is @c true).
4156	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
4157	* enable VT-x on the host.
4158	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
4159	*/
4160	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
4161	PCSUPHWVIRTMSRS pHwvirtMsrs)
4162	{
4163	AssertPtr(pHostCpu);
4164	AssertPtr(pHwvirtMsrs);
4165	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4166
4167	/* Enable VT-x if it's not already enabled by the host. */
4168	if (!fEnabledByHost)
4169	{
4170	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
4171	if (RT_FAILURE(rc))
4172	return rc;
4173	}
4174
4175	/*
4176	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
4177	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
4178	* invalidated when flushing by VPID.
4179	*/
4180	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
4181	{
4182	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
4183	pHostCpu->fFlushAsidBeforeUse = false;
4184	}
4185	else
4186	pHostCpu->fFlushAsidBeforeUse = true;
4187
4188	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
4189	++pHostCpu->cTlbFlushes;
4190
4191	return VINF_SUCCESS;
4192	}
4193
4194
4195	/**
4196	* Deactivates VT-x on the current CPU.
4197	*
4198	* @returns VBox status code.
4199	* @param pvCpuPage Pointer to the VMXON region.
4200	* @param HCPhysCpuPage Physical address of the VMXON region.
4201	*
4202	* @remarks This function should never be called when SUPR0EnableVTx() or
4203	* similar was used to enable VT-x on the host.
4204	*/
4205	VMMR0DECL(int) VMXR0DisableCpu(void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
4206	{
4207	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
4208
4209	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4210	return hmR0VmxLeaveRootMode();
4211	}
4212
4213
4214	/**
4215	* Does per-VM VT-x initialization.
4216	*
4217	* @returns VBox status code.
4218	* @param pVM The cross context VM structure.
4219	*/
4220	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
4221	{
4222	AssertPtr(pVM);
4223	LogFlowFunc(("pVM=%p\n", pVM));
4224
4225	int rc = hmR0VmxStructsAlloc(pVM);
4226	if (RT_FAILURE(rc))
4227	{
4228	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
4229	return rc;
4230	}
4231
4232	return VINF_SUCCESS;
4233	}
4234
4235
4236	/**
4237	* Does per-VM VT-x termination.
4238	*
4239	* @returns VBox status code.
4240	* @param pVM The cross context VM structure.
4241	*/
4242	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
4243	{
4244	AssertPtr(pVM);
4245	LogFlowFunc(("pVM=%p\n", pVM));
4246
4247	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4248	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
4249	{
4250	Assert(pVM->hm.s.vmx.pvScratch);
4251	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
4252	}
4253	#endif
4254	hmR0VmxStructsFree(pVM);
4255	return VINF_SUCCESS;
4256	}
4257
4258
4259	/**
4260	* Sets up the VM for execution using hardware-assisted VMX.
4261	* This function is only called once per-VM during initialization.
4262	*
4263	* @returns VBox status code.
4264	* @param pVM The cross context VM structure.
4265	*/
4266	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
4267	{
4268	AssertPtr(pVM);
4269	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4270
4271	LogFlowFunc(("pVM=%p\n", pVM));
4272
4273	/*
4274	* At least verify if VMX is enabled, since we can't check if we're in
4275	* VMX root mode or not without causing a #GP.
4276	*/
4277	RTCCUINTREG const uHostCr4 = ASMGetCR4();
4278	if (RT_LIKELY(uHostCr4 & X86_CR4_VMXE))
4279	{ /* likely */ }
4280	else
4281	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
4282
4283	/*
4284	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
4285	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
4286	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
4287	*/
4288	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4289	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
4290	\|\| !pVM->hm.s.vmx.pRealModeTSS))
4291	{
4292	LogRelFunc(("Invalid real-on-v86 state.\n"));
4293	return VERR_INTERNAL_ERROR;
4294	}
4295
4296	/* Initialize these always, see hmR3InitFinalizeR0().*/
4297	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
4298	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
4299
4300	/* Setup the tagged-TLB flush handlers. */
4301	int rc = hmR0VmxSetupTaggedTlb(pVM);
4302	if (RT_FAILURE(rc))
4303	{
4304	LogRelFunc(("Failed to setup tagged TLB. rc=%Rrc\n", rc));
4305	return rc;
4306	}
4307
4308	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4309	/* Setup the shadow VMCS fields array and VMREAD/VMWRITE bitmaps. */
4310	if (pVM->hm.s.vmx.fUseVmcsShadowing)
4311	{
4312	rc = hmR0VmxSetupShadowVmcsFieldsArrays(pVM);
4313	if (RT_SUCCESS(rc))
4314	hmR0VmxSetupVmreadVmwriteBitmaps(pVM);
4315	else
4316	{
4317	LogRelFunc(("Failed to setup shadow VMCS fields arrays. rc=%Rrc\n", rc));
4318	return rc;
4319	}
4320	}
4321	#endif
4322
4323	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
4324	{
4325	PVMCPU pVCpu = &pVM->aCpus[idCpu];
4326	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
4327
4328	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
4329	if (RT_SUCCESS(rc))
4330	{
4331	#if HC_ARCH_BITS == 32
4332	hmR0VmxInitVmcsReadCache(pVCpu);
4333	#endif
4334	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4335	if (pVM->cpum.ro.GuestFeatures.fVmx)
4336	{
4337	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
4338	if (RT_SUCCESS(rc))
4339	{ /* likely */ }
4340	else
4341	{
4342	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
4343	return rc;
4344	}
4345	}
4346	#endif
4347	}
4348	else
4349	{
4350	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
4351	return rc;
4352	}
4353	}
4354
4355	return VINF_SUCCESS;
4356	}
4357
4358
4359	/**
4360	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
4361	* the VMCS.
4362	*
4363	* @returns VBox status code.
4364	*/
4365	static int hmR0VmxExportHostControlRegs(void)
4366	{
4367	RTCCUINTREG uReg = ASMGetCR0();
4368	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
4369	AssertRCReturn(rc, rc);
4370
4371	uReg = ASMGetCR3();
4372	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
4373	AssertRCReturn(rc, rc);
4374
4375	uReg = ASMGetCR4();
4376	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
4377	AssertRCReturn(rc, rc);
4378	return rc;
4379	}
4380
4381
4382	/**
4383	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
4384	* the host-state area in the VMCS.
4385	*
4386	* @returns VBox status code.
4387	* @param pVCpu The cross context virtual CPU structure.
4388	*/
4389	static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
4390	{
4391	#if HC_ARCH_BITS == 64
4392	/**
4393	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
4394	* requirements. See hmR0VmxExportHostSegmentRegs().
4395	*/
4396	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
4397	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
4398	{ \
4399	bool fValidSelector = true; \
4400	if ((selValue) & X86_SEL_LDT) \
4401	{ \
4402	uint32_t uAttr = ASMGetSegAttr((selValue)); \
4403	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
4404	} \
4405	if (fValidSelector) \
4406	{ \
4407	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
4408	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
4409	} \
4410	(selValue) = 0; \
4411	}
4412
4413	/*
4414	* If we've executed guest code using hardware-assisted VMX, the host-state bits
4415	* will be messed up. We should -not- save the messed up state without restoring
4416	* the original host-state, see @bugref{7240}.
4417	*
4418	* This apparently can happen (most likely the FPU changes), deal with it rather than
4419	* asserting. Was observed booting Solaris 10u10 32-bit guest.
4420	*/
4421	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
4422	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
4423	{
4424	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
4425	pVCpu->idCpu));
4426	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
4427	}
4428	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
4429	#else
4430	RT_NOREF(pVCpu);
4431	#endif
4432
4433	/*
4434	* Host DS, ES, FS and GS segment registers.
4435	*/
4436	#if HC_ARCH_BITS == 64
4437	RTSEL uSelDS = ASMGetDS();
4438	RTSEL uSelES = ASMGetES();
4439	RTSEL uSelFS = ASMGetFS();
4440	RTSEL uSelGS = ASMGetGS();
4441	#else
4442	RTSEL uSelDS = 0;
4443	RTSEL uSelES = 0;
4444	RTSEL uSelFS = 0;
4445	RTSEL uSelGS = 0;
4446	#endif
4447
4448	/*
4449	* Host CS and SS segment registers.
4450	*/
4451	RTSEL uSelCS = ASMGetCS();
4452	RTSEL uSelSS = ASMGetSS();
4453
4454	/*
4455	* Host TR segment register.
4456	*/
4457	RTSEL uSelTR = ASMGetTR();
4458
4459	#if HC_ARCH_BITS == 64
4460	/*
4461	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
4462	* gain VM-entry and restore them before we get preempted.
4463	*
4464	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
4465	*/
4466	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
4467	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
4468	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
4469	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
4470	# undef VMXLOCAL_ADJUST_HOST_SEG
4471	#endif
4472
4473	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
4474	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
4475	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
4476	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
4477	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
4478	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
4479	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
4480	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
4481	Assert(uSelCS);
4482	Assert(uSelTR);
4483
4484	/* Write these host selector fields into the host-state area in the VMCS. */
4485	int rc = VMXWriteVmcs16(VMX_VMCS16_HOST_CS_SEL, uSelCS);
4486	rc \|= VMXWriteVmcs16(VMX_VMCS16_HOST_SS_SEL, uSelSS);
4487	#if HC_ARCH_BITS == 64
4488	rc \|= VMXWriteVmcs16(VMX_VMCS16_HOST_DS_SEL, uSelDS);
4489	rc \|= VMXWriteVmcs16(VMX_VMCS16_HOST_ES_SEL, uSelES);
4490	rc \|= VMXWriteVmcs16(VMX_VMCS16_HOST_FS_SEL, uSelFS);
4491	rc \|= VMXWriteVmcs16(VMX_VMCS16_HOST_GS_SEL, uSelGS);
4492	#else
4493	NOREF(uSelDS);
4494	NOREF(uSelES);
4495	NOREF(uSelFS);
4496	NOREF(uSelGS);
4497	#endif
4498	rc \|= VMXWriteVmcs16(VMX_VMCS16_HOST_TR_SEL, uSelTR);
4499	AssertRCReturn(rc, rc);
4500
4501	/*
4502	* Host GDTR and IDTR.
4503	*/
4504	RTGDTR Gdtr;
4505	RTIDTR Idtr;
4506	RT_ZERO(Gdtr);
4507	RT_ZERO(Idtr);
4508	ASMGetGDTR(&Gdtr);
4509	ASMGetIDTR(&Idtr);
4510	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
4511	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
4512	AssertRCReturn(rc, rc);
4513
4514	#if HC_ARCH_BITS == 64
4515	/*
4516	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
4517	* them to the maximum limit (0xffff) on every VM-exit.
4518	*/
4519	if (Gdtr.cbGdt != 0xffff)
4520	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
4521
4522	/*
4523	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
4524	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
4525	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
4526	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
4527	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4528	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4529	* at 0xffff on hosts where we are sure it won't cause trouble.
4530	*/
4531	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
4532	if (Idtr.cbIdt < 0x0fff)
4533	# else
4534	if (Idtr.cbIdt != 0xffff)
4535	# endif
4536	{
4537	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4538	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4539	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4540	}
4541	#endif
4542
4543	/*
4544	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4545	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4546	* RPL should be too in most cases.
4547	*/
4548	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4549	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4550
4551	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4552	#if HC_ARCH_BITS == 64
4553	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4554
4555	/*
4556	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4557	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4558	* restoration if the host has something else. Task switching is not supported in 64-bit
4559	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4560	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4561	*
4562	* [1] See Intel spec. 3.5 "System Descriptor Types".
4563	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4564	*/
4565	PVM pVM = pVCpu->CTX_SUFF(pVM);
4566	Assert(pDesc->System.u4Type == 11);
4567	if ( pDesc->System.u16LimitLow != 0x67
4568	\|\| pDesc->System.u4LimitHigh)
4569	{
4570	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4571	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4572	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4573	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4574	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4575	}
4576
4577	/*
4578	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4579	*/
4580	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4581	{
4582	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4583	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4584	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4585	{
4586	/* The GDT is read-only but the writable GDT is available. */
4587	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4588	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4589	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4590	AssertRCReturn(rc, rc);
4591	}
4592	}
4593	#else
4594	uintptr_t const uTRBase = X86DESC_BASE(pDesc);
4595	#endif
4596	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
4597	AssertRCReturn(rc, rc);
4598
4599	/*
4600	* Host FS base and GS base.
4601	*/
4602	#if HC_ARCH_BITS == 64
4603	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4604	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4605	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
4606	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
4607	AssertRCReturn(rc, rc);
4608
4609	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4610	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4611	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4612	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4613	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4614	#endif
4615	return VINF_SUCCESS;
4616	}
4617
4618
4619	/**
4620	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4621	* host-state area of the VMCS.
4622	*
4623	* These MSRs will be automatically restored on the host after every successful
4624	* VM-exit.
4625	*
4626	* @returns VBox status code.
4627	* @param pVCpu The cross context virtual CPU structure.
4628	*
4629	* @remarks No-long-jump zone!!!
4630	*/
4631	static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
4632	{
4633	AssertPtr(pVCpu);
4634
4635	/*
4636	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4637	* rather than swapping them on every VM-entry.
4638	*/
4639	hmR0VmxLazySaveHostMsrs(pVCpu);
4640
4641	/*
4642	* Host Sysenter MSRs.
4643	*/
4644	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
4645	#if HC_ARCH_BITS == 32
4646	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
4647	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
4648	#else
4649	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
4650	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
4651	#endif
4652	AssertRCReturn(rc, rc);
4653
4654	/*
4655	* Host EFER MSR.
4656	*
4657	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4658	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4659	*/
4660	PVM pVM = pVCpu->CTX_SUFF(pVM);
4661	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4662	{
4663	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4664	AssertRCReturn(rc, rc);
4665	}
4666
4667	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4668	* hmR0VmxExportGuestEntryExitCtls(). */
4669
4670	return VINF_SUCCESS;
4671	}
4672
4673
4674	/**
4675	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4676	*
4677	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4678	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4679	*
4680	* @returns true if we need to load guest EFER, false otherwise.
4681	* @param pVCpu The cross context virtual CPU structure.
4682	*
4683	* @remarks Requires EFER, CR4.
4684	* @remarks No-long-jump zone!!!
4685	*/
4686	static bool hmR0VmxShouldSwapEferMsr(PCVMCPU pVCpu)
4687	{
4688	#ifdef HMVMX_ALWAYS_SWAP_EFER
4689	RT_NOREF(pVCpu);
4690	return true;
4691	#else
4692	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4693	PVM pVM = pVCpu->CTX_SUFF(pVM);
4694	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4695	uint64_t const u64GuestEfer = pCtx->msrEFER;
4696
4697	/*
4698	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4699	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4700	*/
4701	if ( CPUMIsGuestInLongModeEx(pCtx)
4702	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4703	return true;
4704
4705	/*
4706	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4707	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4708	*
4709	* See Intel spec. 4.5 "IA-32e Paging".
4710	* See Intel spec. 4.1.1 "Three Paging Modes".
4711	*
4712	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4713	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4714	*/
4715	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4716	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4717	if ( (pCtx->cr4 & X86_CR4_PAE)
4718	&& (pCtx->cr0 & X86_CR0_PG)
4719	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4720	{
4721	/* Assert that host is NX capable. */
4722	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4723	return true;
4724	}
4725
4726	return false;
4727	#endif
4728	}
4729
4730	/**
4731	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4732	* VMCS.
4733	*
4734	* This is typically required when the guest changes paging mode.
4735	*
4736	* @returns VBox status code.
4737	* @param pVCpu The cross context virtual CPU structure.
4738	* @param pVmxTransient The VMX-transient structure.
4739	*
4740	* @remarks Requires EFER.
4741	* @remarks No-long-jump zone!!!
4742	*/
4743	static int hmR0VmxExportGuestEntryExitCtls(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4744	{
4745	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4746	{
4747	PVM pVM = pVCpu->CTX_SUFF(pVM);
4748	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4749
4750	/*
4751	* VM-entry controls.
4752	*/
4753	{
4754	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4755	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4756
4757	/*
4758	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4759	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4760	*
4761	* For nested-guests, this is a mandatory VM-entry control. It's also
4762	* required because we do not want to leak host bits to the nested-guest.
4763	*/
4764	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4765
4766	/*
4767	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4768	*
4769	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4770	* required to get the nested-guest working with hardware-assisted VMX execution.
4771	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a guest hypervisor
4772	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4773	* here rather than while merging the guest VMCS controls.
4774	*/
4775	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4776	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4777	else
4778	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4779
4780	/*
4781	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4782	*
4783	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4784	* regardless of whether the nested-guest VMCS specifies it because we are free to
4785	* load whatever MSRs we require and we do not need to modify the guest visible copy
4786	* of the VM-entry MSR load area.
4787	*/
4788	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4789	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4790	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4791	else
4792	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4793
4794	/*
4795	* The following should -not- be set (since we're not in SMM mode):
4796	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4797	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4798	*/
4799
4800	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4801	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4802
4803	if ((fVal & fZap) == fVal)
4804	{ /* likely */ }
4805	else
4806	{
4807	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
4808	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
4809	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
4810	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4811	}
4812
4813	/* Commit it to the VMCS. */
4814	if (pVmcsInfo->u32EntryCtls != fVal)
4815	{
4816	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
4817	AssertRCReturn(rc, rc);
4818	pVmcsInfo->u32EntryCtls = fVal;
4819	}
4820	}
4821
4822	/*
4823	* VM-exit controls.
4824	*/
4825	{
4826	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4827	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4828
4829	/*
4830	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
4831	* supported the 1-setting of this bit.
4832	*
4833	* For nested-guests, we set the "save debug controls" as the converse
4834	* "load debug controls" is mandatory for nested-guests anyway.
4835	*/
4836	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
4837
4838	/*
4839	* Set the host long mode active (EFER.LMA) bit (which Intel calls
4840	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
4841	* host EFER.LMA and EFER.LME bit to this value. See assertion in
4842	* hmR0VmxExportHostMsrs().
4843	*
4844	* For nested-guests, we always set this bit as we do not support 32-bit
4845	* hosts.
4846	*/
4847	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4848
4849	/*
4850	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
4851	*
4852	* For nested-guests, we should use the "save IA32_EFER" control if we also
4853	* used the "load IA32_EFER" control while exporting VM-entry controls.
4854	*/
4855	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4856	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4857	{
4858	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
4859	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
4860	}
4861
4862	/*
4863	* Enable saving of the VMX-preemption timer value on VM-exit.
4864	* For nested-guests, currently not exposed/used.
4865	*/
4866	if ( pVM->hm.s.vmx.fUsePreemptTimer
4867	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
4868	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
4869
4870	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
4871	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
4872
4873	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
4874	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
4875	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
4876
4877	if ((fVal & fZap) == fVal)
4878	{ /* likely */ }
4879	else
4880	{
4881	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
4882	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
4883	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
4884	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4885	}
4886
4887	/* Commit it to the VMCS. */
4888	if (pVmcsInfo->u32ExitCtls != fVal)
4889	{
4890	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
4891	AssertRCReturn(rc, rc);
4892	pVmcsInfo->u32ExitCtls = fVal;
4893	}
4894	}
4895
4896	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
4897	}
4898	return VINF_SUCCESS;
4899	}
4900
4901
4902	/**
4903	* Sets the TPR threshold in the VMCS.
4904	*
4905	* @returns VBox status code.
4906	* @param pVCpu The cross context virtual CPU structure.
4907	* @param pVmcsInfo The VMCS info. object.
4908	* @param u32TprThreshold The TPR threshold (task-priority class only).
4909	*/
4910	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
4911	{
4912	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
4913	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
4914	RT_NOREF2(pVCpu, pVmcsInfo);
4915	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
4916	}
4917
4918
4919	/**
4920	* Exports the guest APIC TPR state into the VMCS.
4921	*
4922	* @returns VBox status code.
4923	* @param pVCpu The cross context virtual CPU structure.
4924	* @param pVmxTransient The VMX-transient structure.
4925	*
4926	* @remarks No-long-jump zone!!!
4927	*/
4928	static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4929	{
4930	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
4931	{
4932	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
4933
4934	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4935	if (!pVmxTransient->fIsNestedGuest)
4936	{
4937	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
4938	&& APICIsEnabled(pVCpu))
4939	{
4940	/*
4941	* Setup TPR shadowing.
4942	*/
4943	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
4944	{
4945	bool fPendingIntr = false;
4946	uint8_t u8Tpr = 0;
4947	uint8_t u8PendingIntr = 0;
4948	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
4949	AssertRCReturn(rc, rc);
4950
4951	/*
4952	* If there are interrupts pending but masked by the TPR, instruct VT-x to
4953	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
4954	* priority of the pending interrupt so we can deliver the interrupt. If there
4955	* are no interrupts pending, set threshold to 0 to not cause any
4956	* TPR-below-threshold VM-exits.
4957	*/
4958	uint32_t u32TprThreshold = 0;
4959	if (fPendingIntr)
4960	{
4961	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
4962	(which is the Task-Priority Class). */
4963	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
4964	const uint8_t u8TprPriority = u8Tpr >> 4;
4965	if (u8PendingPriority <= u8TprPriority)
4966	u32TprThreshold = u8PendingPriority;
4967	}
4968
4969	rc = hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
4970	AssertRCReturn(rc, rc);
4971	}
4972	}
4973	}
4974	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
4975	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
4976	}
4977	return VINF_SUCCESS;
4978	}
4979
4980
4981	/**
4982	* Gets the guest interruptibility-state.
4983	*
4984	* @returns Guest's interruptibility-state.
4985	* @param pVCpu The cross context virtual CPU structure.
4986	* @param pVmxTransient The VMX-transient structure.
4987	*
4988	* @remarks No-long-jump zone!!!
4989	*/
4990	static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4991	{
4992	/*
4993	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
4994	*/
4995	uint32_t fIntrState = 0;
4996	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4997	{
4998	/* If inhibition is active, RIP and RFLAGS should've been imported from the VMCS already. */
4999	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
5000
5001	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5002	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
5003	{
5004	if (pCtx->eflags.Bits.u1IF)
5005	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
5006	else
5007	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
5008	}
5009	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5010	{
5011	/*
5012	* We can clear the inhibit force flag as even if we go back to the recompiler
5013	* without executing guest code in VT-x, the flag's condition to be cleared is
5014	* met and thus the cleared state is correct.
5015	*/
5016	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5017	}
5018	}
5019
5020	/*
5021	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
5022	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
5023	* setting this would block host-NMIs and IRET will not clear the blocking.
5024	*
5025	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
5026	*
5027	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
5028	*/
5029	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5030	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
5031	&& CPUMIsGuestNmiBlocking(pVCpu))
5032	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
5033
5034	return fIntrState;
5035	}
5036
5037
5038	/**
5039	* Exports the exception intercepts required for guest execution in the VMCS.
5040	*
5041	* @returns VBox status code.
5042	* @param pVCpu The cross context virtual CPU structure.
5043	* @param pVmxTransient The VMX-transient structure.
5044	*
5045	* @remarks No-long-jump zone!!!
5046	*/
5047	static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5048	{
5049	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS)
5050	{
5051	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
5052	if ( !pVmxTransient->fIsNestedGuest
5053	&& pVCpu->hm.s.fGIMTrapXcptUD)
5054	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
5055	else
5056	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
5057
5058	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
5059	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS);
5060	}
5061	return VINF_SUCCESS;
5062	}
5063
5064
5065	/**
5066	* Exports the guest's RIP into the guest-state area in the VMCS.
5067	*
5068	* @returns VBox status code.
5069	* @param pVCpu The cross context virtual CPU structure.
5070	*
5071	* @remarks No-long-jump zone!!!
5072	*/
5073	static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
5074	{
5075	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
5076	{
5077	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
5078
5079	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
5080	AssertRCReturn(rc, rc);
5081
5082	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
5083	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
5084	}
5085	return VINF_SUCCESS;
5086	}
5087
5088
5089	/**
5090	* Exports the guest's RSP into the guest-state area in the VMCS.
5091	*
5092	* @returns VBox status code.
5093	* @param pVCpu The cross context virtual CPU structure.
5094	*
5095	* @remarks No-long-jump zone!!!
5096	*/
5097	static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
5098	{
5099	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
5100	{
5101	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
5102
5103	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
5104	AssertRCReturn(rc, rc);
5105
5106	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
5107	}
5108	return VINF_SUCCESS;
5109	}
5110
5111
5112	/**
5113	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
5114	*
5115	* @returns VBox status code.
5116	* @param pVCpu The cross context virtual CPU structure.
5117	* @param pVmxTransient The VMX-transient structure.
5118	*
5119	* @remarks No-long-jump zone!!!
5120	*/
5121	static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5122	{
5123	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
5124	{
5125	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
5126
5127	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
5128	Let us assert it as such and use 32-bit VMWRITE. */
5129	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
5130	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
5131	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
5132	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
5133
5134	/*
5135	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
5136	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
5137	* can run the real-mode guest code under Virtual 8086 mode.
5138	*/
5139	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5140	if (pVmcsInfo->RealMode.fRealOnV86Active)
5141	{
5142	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5143	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5144	Assert(!pVmxTransient->fIsNestedGuest);
5145	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
5146	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
5147	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
5148	}
5149
5150	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
5151	AssertRCReturn(rc, rc);
5152
5153	/*
5154	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
5155	*
5156	* We must avoid setting any automatic debug exceptions delivery when single-stepping
5157	* through the hypervisor debugger using EFLAGS.TF.
5158	*/
5159	if ( !pVmxTransient->fIsNestedGuest
5160	&& !pVCpu->hm.s.fSingleInstruction
5161	&& fEFlags.Bits.u1TF)
5162	{
5163	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
5164	* premature trips to ring-3 esp since IEM does not yet handle it. */
5165	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
5166	AssertRCReturn(rc, rc);
5167	}
5168	/** @todo NSTVMX: Handling copying of VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS from
5169	* nested-guest VMCS. */
5170
5171	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
5172	Log4Func(("eflags=%#RX32\n", fEFlags.u32));
5173	}
5174	return VINF_SUCCESS;
5175	}
5176
5177
5178	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5179	/**
5180	* Copies the nested-guest VMCS to the shadow VMCS.
5181	*
5182	* @returns VBox status code.
5183	* @param pVCpu The cross context virtual CPU structure.
5184	* @param pVmcsInfo The VMCS info. object.
5185	*
5186	* @remarks No-long-jump zone!!!
5187	*/
5188	static int hmR0VmxCopyNstGstToShadowVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5189	{
5190	PVM pVM = pVCpu->CTX_SUFF(pVM);
5191	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
5192
5193	/*
5194	* Disable interrupts so we don't get preempted while the shadow VMCS is the
5195	* current VMCS, as we may try saving guest lazy MSRs.
5196	*
5197	* Strictly speaking the lazy MSRs are not in the VMCS, but I'd rather not risk
5198	* calling the import VMCS code which is currently performing the guest MSR reads
5199	* (on 64-bit hosts) and accessing the auto-load/store MSR area on 32-bit hosts
5200	* and the rest of the VMX leave session machinery.
5201	*/
5202	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
5203
5204	int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
5205	if (RT_SUCCESS(rc))
5206	{
5207	/*
5208	* Copy all guest read/write VMCS fields.
5209	*
5210	* We don't check for VMWRITE failures here for performance reasons and
5211	* because they are not expected to fail, barring irrecoverable conditions
5212	* like hardware errors.
5213	*/
5214	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
5215	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
5216	{
5217	uint64_t u64Val;
5218	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
5219	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
5220	VMXWriteVmcs64(uVmcsField, u64Val);
5221	}
5222
5223	/*
5224	* If the host CPU supports writing all VMCS fields, copy the guest read-only
5225	* VMCS fields, so the guest can VMREAD them without causing a VM-exit.
5226	*/
5227	if (pVM->hm.s.vmx.Msrs.u64Misc & VMX_MISC_VMWRITE_ALL)
5228	{
5229	uint32_t const cShadowVmcsRoFields = pVM->hm.s.vmx.cShadowVmcsRoFields;
5230	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
5231	{
5232	uint64_t u64Val;
5233	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsRoFields[i];
5234	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
5235	VMXWriteVmcs64(uVmcsField, u64Val);
5236	}
5237	}
5238
5239	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
5240	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
5241	}
5242
5243	ASMSetFlags(fEFlags);
5244	return rc;
5245	}
5246
5247
5248	/**
5249	* Copies the shadow VMCS to the nested-guest VMCS.
5250	*
5251	* @returns VBox status code.
5252	* @param pVCpu The cross context virtual CPU structure.
5253	* @param pVmcsInfo The VMCS info. object.
5254	*
5255	* @remarks Called with interrupts disabled.
5256	*/
5257	static int hmR0VmxCopyShadowToNstGstVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5258	{
5259	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5260	PVM pVM = pVCpu->CTX_SUFF(pVM);
5261	PVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
5262
5263	int rc = hmR0VmxLoadShadowVmcs(pVmcsInfo);
5264	if (RT_SUCCESS(rc))
5265	{
5266	/*
5267	* Copy guest read/write fields from the shadow VMCS.
5268	* Guest read-only fields cannot be modified, so no need to copy them.
5269	*
5270	* We don't check for VMREAD failures here for performance reasons and
5271	* because they are not expected to fail, barring irrecoverable conditions
5272	* like hardware errors.
5273	*/
5274	uint32_t const cShadowVmcsFields = pVM->hm.s.vmx.cShadowVmcsFields;
5275	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
5276	{
5277	uint64_t u64Val;
5278	uint32_t const uVmcsField = pVM->hm.s.vmx.paShadowVmcsFields[i];
5279	VMXReadVmcs64(uVmcsField, &u64Val);
5280	IEMWriteVmxVmcsField(pVmcsNstGst, uVmcsField, u64Val);
5281	}
5282
5283	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
5284	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
5285	}
5286	return rc;
5287	}
5288
5289
5290	/**
5291	* Enables VMCS shadowing for the given VMCS info. object.
5292	*
5293	* @param pVCpu The cross context virtual CPU structure.
5294	* @param pVmcsInfo The VMCS info. object.
5295	*
5296	* @remarks No-long-jump zone!!!
5297	*/
5298	static void hmR0VmxEnableVmcsShadowing(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5299	{
5300	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
5301
5302	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
5303	if (!(uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING))
5304	{
5305	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
5306	uProcCtls2 \|= VMX_PROC_CTLS2_VMCS_SHADOWING;
5307	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2);
5308	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, pVmcsInfo->HCPhysShadowVmcs);
5309	AssertRC(rc);
5310
5311	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
5312	pVmcsInfo->u64VmcsLinkPtr = pVmcsInfo->HCPhysShadowVmcs;
5313	Log4Func(("Enabled\n"));
5314	}
5315	}
5316
5317
5318	/**
5319	* Disables VMCS shadowing for the given VMCS info. object.
5320	*
5321	* @param pVCpu The cross context virtual CPU structure.
5322	* @param pVmcsInfo The VMCS info. object.
5323	*
5324	* @remarks No-long-jump zone!!!
5325	*/
5326	static void hmR0VmxDisableVmcsShadowing(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
5327	{
5328	NOREF(pVCpu); /* Used implicitly by VMXWriteVmcs64 on 32-bit hosts. */
5329
5330	/*
5331	* We want all VMREAD and VMWRITE instructions to cause VM-exits, so we clear the
5332	* VMCS shadowing control. However, VM-entry requires the shadow VMCS indicator bit
5333	* to match the VMCS shadowing control if the VMCS link pointer is not NIL_RTHCPHYS.
5334	* Hence, we must also reset the VMCS link pointer to ensure VM-entry does not fail.
5335	*
5336	* See Intel spec. 26.2.1.1 "VM-Execution Control Fields".
5337	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5338	*/
5339	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
5340	if (uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
5341	{
5342	uProcCtls2 &= ~VMX_PROC_CTLS2_VMCS_SHADOWING;
5343	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2);
5344	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);
5345	AssertRC(rc);
5346
5347	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
5348	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
5349	Log4Func(("Disabled\n"));
5350	}
5351	}
5352	#endif
5353
5354
5355	/**
5356	* Exports the guest hardware-virtualization state.
5357	*
5358	* @returns VBox status code.
5359	* @param pVCpu The cross context virtual CPU structure.
5360	* @param pVmxTransient The VMX-transient structure.
5361	*
5362	* @remarks No-long-jump zone!!!
5363	*/
5364	static int hmR0VmxExportGuestHwvirtState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5365	{
5366	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_HWVIRT)
5367	{
5368	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5369	/*
5370	* Check if the VMX feature is exposed to the guest and if the host CPU supports
5371	* VMCS shadowing.
5372	*/
5373	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUseVmcsShadowing)
5374	{
5375	/*
5376	* If the guest hypervisor has loaded a current VMCS and is in VMX root mode,
5377	* copy the guest hypervisor's current VMCS into the shadow VMCS and enable
5378	* VMCS shadowing to skip intercepting some or all VMREAD/VMWRITE VM-exits.
5379	*
5380	* We check for VMX root mode here in case the guest executes VMXOFF without
5381	* clearing the current VMCS pointer and our VMXOFF instruction emulation does
5382	* not clear the current VMCS pointer.
5383	*/
5384	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5385	if ( CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx)
5386	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)
5387	&& CPUMIsGuestVmxCurrentVmcsValid(pVCpu, &pVCpu->cpum.GstCtx))
5388	{
5389	/* Paranoia. */
5390	Assert(!pVmxTransient->fIsNestedGuest);
5391
5392	/*
5393	* For performance reasons, also check if the guest hypervisor's current VMCS
5394	* was newly loaded or modified before copying it to the shadow VMCS.
5395	*/
5396	if (!pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs)
5397	{
5398	int rc = hmR0VmxCopyNstGstToShadowVmcs(pVCpu, pVmcsInfo);
5399	AssertRCReturn(rc, rc);
5400	pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs = true;
5401	}
5402	hmR0VmxEnableVmcsShadowing(pVCpu, pVmcsInfo);
5403	}
5404	else
5405	hmR0VmxDisableVmcsShadowing(pVCpu, pVmcsInfo);
5406	}
5407	#else
5408	NOREF(pVmxTransient);
5409	#endif
5410	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_HWVIRT);
5411	}
5412	return VINF_SUCCESS;
5413	}
5414
5415
5416	/**
5417	* Exports the guest CR0 control register into the guest-state area in the VMCS.
5418	*
5419	* The guest FPU state is always pre-loaded hence we don't need to bother about
5420	* sharing FPU related CR0 bits between the guest and host.
5421	*
5422	* @returns VBox status code.
5423	* @param pVCpu The cross context virtual CPU structure.
5424	* @param pVmxTransient The VMX-transient structure.
5425	*
5426	* @remarks No-long-jump zone!!!
5427	*/
5428	static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5429	{
5430	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
5431	{
5432	PVM pVM = pVCpu->CTX_SUFF(pVM);
5433	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5434
5435	/*
5436	* Figure out fixed CR0 bits in VMX operation.
5437	*/
5438	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
5439	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
5440	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
5441	if (pVM->hm.s.vmx.fUnrestrictedGuest)
5442	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
5443	else
5444	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
5445
5446	if (!pVmxTransient->fIsNestedGuest)
5447	{
5448	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5449	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5450	uint64_t const u64ShadowCr0 = u64GuestCr0;
5451	Assert(!RT_HI_U32(u64GuestCr0));
5452
5453	/*
5454	* Setup VT-x's view of the guest CR0.
5455	*/
5456	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5457	if (pVM->hm.s.fNestedPaging)
5458	{
5459	if (CPUMIsGuestPagingEnabled(pVCpu))
5460	{
5461	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
5462	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
5463	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
5464	}
5465	else
5466	{
5467	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
5468	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
5469	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
5470	}
5471
5472	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
5473	if (pVM->hm.s.vmx.fUnrestrictedGuest)
5474	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
5475	}
5476	else
5477	{
5478	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
5479	u64GuestCr0 \|= X86_CR0_WP;
5480	}
5481
5482	/*
5483	* Guest FPU bits.
5484	*
5485	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
5486	* using CR0.TS.
5487	*
5488	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
5489	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
5490	*/
5491	u64GuestCr0 \|= X86_CR0_NE;
5492
5493	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
5494	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
5495
5496	/*
5497	* Update exception intercepts.
5498	*/
5499	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
5500	if (pVmcsInfo->RealMode.fRealOnV86Active)
5501	{
5502	Assert(PDMVmmDevHeapIsEnabled(pVM));
5503	Assert(pVM->hm.s.vmx.pRealModeTSS);
5504	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
5505	}
5506	else
5507	{
5508	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
5509	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
5510	if (fInterceptMF)
5511	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
5512	}
5513
5514	/* Additional intercepts for debugging, define these yourself explicitly. */
5515	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
5516	uXcptBitmap \|= 0
5517	\| RT_BIT(X86_XCPT_BP)
5518	\| RT_BIT(X86_XCPT_DE)
5519	\| RT_BIT(X86_XCPT_NM)
5520	\| RT_BIT(X86_XCPT_TS)
5521	\| RT_BIT(X86_XCPT_UD)
5522	\| RT_BIT(X86_XCPT_NP)
5523	\| RT_BIT(X86_XCPT_SS)
5524	\| RT_BIT(X86_XCPT_GP)
5525	\| RT_BIT(X86_XCPT_PF)
5526	\| RT_BIT(X86_XCPT_MF)
5527	;
5528	#elif defined(HMVMX_ALWAYS_TRAP_PF)
5529	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
5530	#endif
5531	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
5532	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
5533	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
5534
5535	/* Apply the hardware specified fixed CR0 bits and enable caching. */
5536	u64GuestCr0 \|= fSetCr0;
5537	u64GuestCr0 &= fZapCr0;
5538	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5539
5540	/* Commit the CR0 and related fields to the guest VMCS. */
5541	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0); /** @todo Fix to 64-bit when we drop 32-bit. */
5542	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
5543	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5544	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5545	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
5546	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
5547	AssertRCReturn(rc, rc);
5548
5549	/* Update our caches. */
5550	pVmcsInfo->u32ProcCtls = uProcCtls;
5551	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
5552
5553	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5554	}
5555	else
5556	{
5557	/*
5558	* With nested-guests, we may have extended the guest/host mask here since we
5559	* merged in the outer guest's mask. Thus, the merged mask can include more bits
5560	* (to read from the nested-guest CR0 read-shadow) than the guest hypervisor
5561	* originally supplied. We must copy those bits from the nested-guest CR0 into
5562	* the nested-guest CR0 read-shadow.
5563	*/
5564	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
5565	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
5566	uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(pVCpu, &pVCpu->cpum.GstCtx, pVmcsInfo->u64Cr0Mask);
5567	Assert(!RT_HI_U32(u64GuestCr0));
5568	Assert(u64GuestCr0 & X86_CR0_NE);
5569
5570	/*
5571	* Apply the hardware specified fixed CR0 bits and enable caching.
5572	* Note! We could be altering our VMX emulation's fixed bits. We thus
5573	* need to re-apply them while importing CR0.
5574	*/
5575	u64GuestCr0 \|= fSetCr0;
5576	u64GuestCr0 &= fZapCr0;
5577	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
5578
5579	/* Commit the CR0 and CR0 read-shadow to the nested-guest VMCS. */
5580	/** @todo NSTVMX: Fix to 64-bit when we drop 32-bit. */
5581	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u64GuestCr0);
5582	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
5583	AssertRCReturn(rc, rc);
5584
5585	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
5586	}
5587
5588	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
5589	}
5590
5591	return VINF_SUCCESS;
5592	}
5593
5594
5595	/**
5596	* Exports the guest control registers (CR3, CR4) into the guest-state area
5597	* in the VMCS.
5598	*
5599	* @returns VBox strict status code.
5600	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
5601	* without unrestricted guest access and the VMMDev is not presently
5602	* mapped (e.g. EFI32).
5603	*
5604	* @param pVCpu The cross context virtual CPU structure.
5605	* @param pVmxTransient The VMX-transient structure.
5606	*
5607	* @remarks No-long-jump zone!!!
5608	*/
5609	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5610	{
5611	int rc = VINF_SUCCESS;
5612	PVM pVM = pVCpu->CTX_SUFF(pVM);
5613
5614	/*
5615	* Guest CR2.
5616	* It's always loaded in the assembler code. Nothing to do here.
5617	*/
5618
5619	/*
5620	* Guest CR3.
5621	*/
5622	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
5623	{
5624	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
5625
5626	RTGCPHYS GCPhysGuestCr3 = NIL_RTGCPHYS;
5627	if (pVM->hm.s.fNestedPaging)
5628	{
5629	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5630	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
5631
5632	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
5633	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
5634	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
5635	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
5636
5637	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
5638	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
5639	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
5640
5641	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
5642	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
5643	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
5644	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
5645	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
5646	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
5647	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
5648
5649	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
5650	AssertRCReturn(rc, rc);
5651
5652	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5653	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5654	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
5655	{
5656	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
5657	if (CPUMIsGuestInPAEModeEx(pCtx))
5658	{
5659	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5660	AssertRCReturn(rc, rc);
5661	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
5662	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
5663	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
5664	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
5665	AssertRCReturn(rc, rc);
5666	}
5667
5668	/*
5669	* The guest's view of its CR3 is unblemished with nested paging when the
5670	* guest is using paging or we have unrestricted guest execution to handle
5671	* the guest when it's not using paging.
5672	*/
5673	GCPhysGuestCr3 = pCtx->cr3;
5674	}
5675	else
5676	{
5677	/*
5678	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
5679	* thinks it accesses physical memory directly, we use our identity-mapped
5680	* page table to map guest-linear to guest-physical addresses. EPT takes care
5681	* of translating it to host-physical addresses.
5682	*/
5683	RTGCPHYS GCPhys;
5684	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
5685
5686	/* We obtain it here every time as the guest could have relocated this PCI region. */
5687	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
5688	if (RT_SUCCESS(rc))
5689	{ /* likely */ }
5690	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
5691	{
5692	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
5693	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
5694	}
5695	else
5696	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
5697
5698	GCPhysGuestCr3 = GCPhys;
5699	}
5700
5701	Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCr3));
5702	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCr3);
5703	AssertRCReturn(rc, rc);
5704	}
5705	else
5706	{
5707	/* Non-nested paging case, just use the hypervisor's CR3. */
5708	RTHCPHYS const HCPhysGuestCr3 = PGMGetHyperCR3(pVCpu);
5709
5710	Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCr3));
5711	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCr3);
5712	AssertRCReturn(rc, rc);
5713	}
5714
5715	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
5716	}
5717
5718	/*
5719	* Guest CR4.
5720	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
5721	*/
5722	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
5723	{
5724	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5725	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5726
5727	/*
5728	* Figure out fixed CR4 bits in VMX operation.
5729	*/
5730	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
5731	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5732	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
5733
5734	/*
5735	* With nested-guests, we may have extended the guest/host mask here (since we
5736	* merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
5737	* mask can include more bits (to read from the nested-guest CR4 read-shadow) than
5738	* the guest hypervisor originally supplied. Thus, we should, in essence, copy
5739	* those bits from the nested-guest CR4 into the nested-guest CR4 read-shadow.
5740	*/
5741	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
5742	uint64_t u64GuestCr4 = pCtx->cr4;
5743	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest
5744	? pCtx->cr4
5745	: CPUMGetGuestVmxMaskedCr4(pVCpu, pCtx, pVmcsInfo->u64Cr4Mask);
5746	Assert(!RT_HI_U32(u64GuestCr4));
5747
5748	/*
5749	* Setup VT-x's view of the guest CR4.
5750	*
5751	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
5752	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
5753	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
5754	*
5755	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
5756	*/
5757	if (pVmcsInfo->RealMode.fRealOnV86Active)
5758	{
5759	Assert(pVM->hm.s.vmx.pRealModeTSS);
5760	Assert(PDMVmmDevHeapIsEnabled(pVM));
5761	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5762	}
5763
5764	if (pVM->hm.s.fNestedPaging)
5765	{
5766	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5767	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
5768	{
5769	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5770	u64GuestCr4 \|= X86_CR4_PSE;
5771	/* Our identity mapping is a 32-bit page directory. */
5772	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5773	}
5774	/* else use guest CR4.*/
5775	}
5776	else
5777	{
5778	Assert(!pVmxTransient->fIsNestedGuest);
5779
5780	/*
5781	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
5782	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
5783	*/
5784	switch (pVCpu->hm.s.enmShadowMode)
5785	{
5786	case PGMMODE_REAL: /* Real-mode. */
5787	case PGMMODE_PROTECTED: /* Protected mode without paging. */
5788	case PGMMODE_32_BIT: /* 32-bit paging. */
5789	{
5790	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5791	break;
5792	}
5793
5794	case PGMMODE_PAE: /* PAE paging. */
5795	case PGMMODE_PAE_NX: /* PAE paging with NX. */
5796	{
5797	u64GuestCr4 \|= X86_CR4_PAE;
5798	break;
5799	}
5800
5801	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
5802	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
5803	#ifdef VBOX_WITH_64_BITS_GUESTS
5804	break;
5805	#endif
5806	default:
5807	AssertFailed();
5808	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5809	}
5810	}
5811
5812	/*
5813	* Apply the hardware specified fixed CR4 bits (mainly CR4.VMXE).
5814	* Note! For nested-guests, we could be altering our VMX emulation's
5815	* fixed bits. We thus need to re-apply them while importing CR4.
5816	*/
5817	u64GuestCr4 \|= fSetCr4;
5818	u64GuestCr4 &= fZapCr4;
5819
5820	/* Commit the CR4 and CR4 read-shadow to the guest VMCS. */
5821	/** @todo Fix to 64-bit when we drop 32-bit. */
5822	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u64GuestCr4);
5823	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4);
5824	AssertRCReturn(rc, rc);
5825
5826	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
5827	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
5828
5829	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
5830
5831	Log4Func(("cr4=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
5832	}
5833	return rc;
5834	}
5835
5836
5837	/**
5838	* Exports the guest debug registers into the guest-state area in the VMCS.
5839	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
5840	*
5841	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
5842	*
5843	* @returns VBox status code.
5844	* @param pVCpu The cross context virtual CPU structure.
5845	* @param pVmxTransient The VMX-transient structure.
5846	*
5847	* @remarks No-long-jump zone!!!
5848	*/
5849	static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5850	{
5851	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5852
5853	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
5854	* stepping. */
5855	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5856	if (pVmxTransient->fIsNestedGuest)
5857	{
5858	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
5859	AssertRCReturn(rc, rc);
5860
5861	/* Always intercept Mov DRx accesses for the nested-guest for now. */
5862	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5863	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
5864	AssertRCReturn(rc, rc);
5865	return VINF_SUCCESS;
5866	}
5867
5868	#ifdef VBOX_STRICT
5869	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
5870	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
5871	{
5872	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
5873	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
5874	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
5875	}
5876	#endif
5877
5878	bool fSteppingDB = false;
5879	bool fInterceptMovDRx = false;
5880	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5881	if (pVCpu->hm.s.fSingleInstruction)
5882	{
5883	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
5884	PVM pVM = pVCpu->CTX_SUFF(pVM);
5885	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
5886	{
5887	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
5888	Assert(fSteppingDB == false);
5889	}
5890	else
5891	{
5892	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
5893	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
5894	pVCpu->hm.s.fClearTrapFlag = true;
5895	fSteppingDB = true;
5896	}
5897	}
5898
5899	uint32_t u32GuestDr7;
5900	if ( fSteppingDB
5901	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
5902	{
5903	/*
5904	* Use the combined guest and host DRx values found in the hypervisor register set
5905	* because the hypervisor debugger has breakpoints active or someone is single stepping
5906	* on the host side without a monitor trap flag.
5907	*
5908	* Note! DBGF expects a clean DR6 state before executing guest code.
5909	*/
5910	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5911	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5912	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
5913	{
5914	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5915	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
5916	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
5917	}
5918	else
5919	#endif
5920	if (!CPUMIsHyperDebugStateActive(pVCpu))
5921	{
5922	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5923	Assert(CPUMIsHyperDebugStateActive(pVCpu));
5924	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5925	}
5926
5927	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
5928	u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
5929	pVCpu->hm.s.fUsingHyperDR7 = true;
5930	fInterceptMovDRx = true;
5931	}
5932	else
5933	{
5934	/*
5935	* If the guest has enabled debug registers, we need to load them prior to
5936	* executing guest code so they'll trigger at the right time.
5937	*/
5938	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
5939	{
5940	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5941	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5942	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
5943	{
5944	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5945	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
5946	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
5947	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5948	}
5949	else
5950	#endif
5951	if (!CPUMIsGuestDebugStateActive(pVCpu))
5952	{
5953	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5954	Assert(CPUMIsGuestDebugStateActive(pVCpu));
5955	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5956	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5957	}
5958	Assert(!fInterceptMovDRx);
5959	}
5960	/*
5961	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
5962	* must intercept #DB in order to maintain a correct DR6 guest value, and
5963	* because we need to intercept it to prevent nested #DBs from hanging the
5964	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
5965	*/
5966	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5967	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
5968	&& !CPUMIsGuestDebugStateActive(pVCpu))
5969	#else
5970	else if (!CPUMIsGuestDebugStateActive(pVCpu))
5971	#endif
5972	{
5973	fInterceptMovDRx = true;
5974	}
5975
5976	/* Update DR7 with the actual guest value. */
5977	u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
5978	pVCpu->hm.s.fUsingHyperDR7 = false;
5979	}
5980
5981	if (fInterceptMovDRx)
5982	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5983	else
5984	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
5985
5986	/*
5987	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
5988	* monitor-trap flag and update our cache.
5989	*/
5990	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5991	{
5992	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5993	AssertRCReturn(rc2, rc2);
5994	pVmcsInfo->u32ProcCtls = uProcCtls;
5995	}
5996
5997	/*
5998	* Update guest DR7.
5999	*/
6000	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
6001	AssertRCReturn(rc, rc);
6002
6003	/*
6004	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
6005	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
6006	*
6007	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
6008	*/
6009	if (fSteppingDB)
6010	{
6011	Assert(pVCpu->hm.s.fSingleInstruction);
6012	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
6013
6014	uint32_t fIntrState = 0;
6015	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
6016	AssertRCReturn(rc, rc);
6017
6018	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
6019	{
6020	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
6021	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
6022	AssertRCReturn(rc, rc);
6023	}
6024	}
6025
6026	return VINF_SUCCESS;
6027	}
6028
6029
6030	#ifdef VBOX_STRICT
6031	/**
6032	* Strict function to validate segment registers.
6033	*
6034	* @param pVCpu The cross context virtual CPU structure.
6035	* @param pVmcsInfo The VMCS info. object.
6036	*
6037	* @remarks Will import guest CR0 on strict builds during validation of
6038	* segments.
6039	*/
6040	static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
6041	{
6042	/*
6043	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6044	*
6045	* The reason we check for attribute value 0 in this function and not just the unusable bit is
6046	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
6047	* unusable bit and doesn't change the guest-context value.
6048	*/
6049	PVM pVM = pVCpu->CTX_SUFF(pVM);
6050	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6051	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
6052	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6053	&& ( !CPUMIsGuestInRealModeEx(pCtx)
6054	&& !CPUMIsGuestInV86ModeEx(pCtx)))
6055	{
6056	/* Protected mode checks */
6057	/* CS */
6058	Assert(pCtx->cs.Attr.n.u1Present);
6059	Assert(!(pCtx->cs.Attr.u & 0xf00));
6060	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
6061	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
6062	\|\| !(pCtx->cs.Attr.n.u1Granularity));
6063	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
6064	\|\| (pCtx->cs.Attr.n.u1Granularity));
6065	/* CS cannot be loaded with NULL in protected mode. */
6066	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
6067	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
6068	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
6069	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
6070	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
6071	else
6072	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
6073	/* SS */
6074	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
6075	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
6076	if ( !(pCtx->cr0 & X86_CR0_PE)
6077	\|\| pCtx->cs.Attr.n.u4Type == 3)
6078	{
6079	Assert(!pCtx->ss.Attr.n.u2Dpl);
6080	}
6081	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
6082	{
6083	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
6084	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
6085	Assert(pCtx->ss.Attr.n.u1Present);
6086	Assert(!(pCtx->ss.Attr.u & 0xf00));
6087	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
6088	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
6089	\|\| !(pCtx->ss.Attr.n.u1Granularity));
6090	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
6091	\|\| (pCtx->ss.Attr.n.u1Granularity));
6092	}
6093	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
6094	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
6095	{
6096	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6097	Assert(pCtx->ds.Attr.n.u1Present);
6098	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
6099	Assert(!(pCtx->ds.Attr.u & 0xf00));
6100	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
6101	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
6102	\|\| !(pCtx->ds.Attr.n.u1Granularity));
6103	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
6104	\|\| (pCtx->ds.Attr.n.u1Granularity));
6105	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6106	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
6107	}
6108	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
6109	{
6110	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6111	Assert(pCtx->es.Attr.n.u1Present);
6112	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
6113	Assert(!(pCtx->es.Attr.u & 0xf00));
6114	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
6115	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
6116	\|\| !(pCtx->es.Attr.n.u1Granularity));
6117	Assert( !(pCtx->es.u32Limit & 0xfff00000)
6118	\|\| (pCtx->es.Attr.n.u1Granularity));
6119	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6120	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
6121	}
6122	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
6123	{
6124	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6125	Assert(pCtx->fs.Attr.n.u1Present);
6126	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
6127	Assert(!(pCtx->fs.Attr.u & 0xf00));
6128	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
6129	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
6130	\|\| !(pCtx->fs.Attr.n.u1Granularity));
6131	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
6132	\|\| (pCtx->fs.Attr.n.u1Granularity));
6133	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6134	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
6135	}
6136	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
6137	{
6138	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
6139	Assert(pCtx->gs.Attr.n.u1Present);
6140	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
6141	Assert(!(pCtx->gs.Attr.u & 0xf00));
6142	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
6143	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
6144	\|\| !(pCtx->gs.Attr.n.u1Granularity));
6145	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
6146	\|\| (pCtx->gs.Attr.n.u1Granularity));
6147	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
6148	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
6149	}
6150	/* 64-bit capable CPUs. */
6151	# if HC_ARCH_BITS == 64
6152	Assert(!RT_HI_U32(pCtx->cs.u64Base));
6153	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
6154	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
6155	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
6156	# endif
6157	}
6158	else if ( CPUMIsGuestInV86ModeEx(pCtx)
6159	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
6160	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
6161	{
6162	/* Real and v86 mode checks. */
6163	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
6164	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
6165	if (pVmcsInfo->RealMode.fRealOnV86Active)
6166	{
6167	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
6168	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
6169	}
6170	else
6171	{
6172	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
6173	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
6174	}
6175
6176	/* CS */
6177	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
6178	Assert(pCtx->cs.u32Limit == 0xffff);
6179	Assert(u32CSAttr == 0xf3);
6180	/* SS */
6181	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
6182	Assert(pCtx->ss.u32Limit == 0xffff);
6183	Assert(u32SSAttr == 0xf3);
6184	/* DS */
6185	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
6186	Assert(pCtx->ds.u32Limit == 0xffff);
6187	Assert(u32DSAttr == 0xf3);
6188	/* ES */
6189	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
6190	Assert(pCtx->es.u32Limit == 0xffff);
6191	Assert(u32ESAttr == 0xf3);
6192	/* FS */
6193	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
6194	Assert(pCtx->fs.u32Limit == 0xffff);
6195	Assert(u32FSAttr == 0xf3);
6196	/* GS */
6197	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
6198	Assert(pCtx->gs.u32Limit == 0xffff);
6199	Assert(u32GSAttr == 0xf3);
6200	/* 64-bit capable CPUs. */
6201	# if HC_ARCH_BITS == 64
6202	Assert(!RT_HI_U32(pCtx->cs.u64Base));
6203	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
6204	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
6205	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
6206	# endif
6207	}
6208	}
6209	#endif /* VBOX_STRICT */
6210
6211
6212	/**
6213	* Exports a guest segment register into the guest-state area in the VMCS.
6214	*
6215	* @returns VBox status code.
6216	* @param pVCpu The cross context virtual CPU structure.
6217	* @param pVmcsInfo The VMCS info. object.
6218	* @param iSegReg The segment register number (X86_SREG_XXX).
6219	* @param pSelReg Pointer to the segment selector.
6220	*
6221	* @remarks No-long-jump zone!!!
6222	*/
6223	static int hmR0VmxExportGuestSegReg(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
6224	{
6225	Assert(iSegReg < X86_SREG_COUNT);
6226	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
6227	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
6228	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
6229	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
6230
6231	uint32_t u32Access = pSelReg->Attr.u;
6232	if (pVmcsInfo->RealMode.fRealOnV86Active)
6233	{
6234	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
6235	u32Access = 0xf3;
6236	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6237	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
6238	RT_NOREF_PV(pVCpu);
6239	}
6240	else
6241	{
6242	/*
6243	* The way to differentiate between whether this is really a null selector or was just
6244	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
6245	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
6246	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
6247	* NULL selectors loaded in protected-mode have their attribute as 0.
6248	*/
6249	if (!u32Access)
6250	u32Access = X86DESCATTR_UNUSABLE;
6251	}
6252
6253	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
6254	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
6255	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
6256
6257	/*
6258	* Commit it to the VMCS.
6259	*/
6260	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel);
6261	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit);
6262	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base);
6263	rc \|= VMXWriteVmcs32(idxAttr, u32Access);
6264	AssertRCReturn(rc, rc);
6265	return rc;
6266	}
6267
6268
6269	/**
6270	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
6271	* area in the VMCS.
6272	*
6273	* @returns VBox status code.
6274	* @param pVCpu The cross context virtual CPU structure.
6275	* @param pVmxTransient The VMX-transient structure.
6276	*
6277	* @remarks Will import guest CR0 on strict builds during validation of
6278	* segments.
6279	* @remarks No-long-jump zone!!!
6280	*/
6281	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6282	{
6283	int rc = VERR_INTERNAL_ERROR_5;
6284	PVM pVM = pVCpu->CTX_SUFF(pVM);
6285	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6286	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6287
6288	/*
6289	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
6290	*/
6291	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
6292	{
6293	#ifdef VBOX_WITH_REM
6294	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6295	{
6296	Assert(!pVmxTransient->fIsNestedGuest);
6297	Assert(pVM->hm.s.vmx.pRealModeTSS);
6298	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
6299	if ( pVmcsInfo->fWasInRealMode
6300	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
6301	{
6302	/*
6303	* Notify the recompiler must flush its code-cache as the guest -may-
6304	* rewrite code it in real-mode (e.g. OpenBSD 4.0).
6305	*/
6306	REMFlushTBs(pVM);
6307	Log4Func(("Switch to protected mode detected!\n"));
6308	pVmcsInfo->fWasInRealMode = false;
6309	}
6310	}
6311	#endif
6312	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
6313	{
6314	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
6315	if (pVmcsInfo->RealMode.fRealOnV86Active)
6316	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
6317	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
6318	AssertRCReturn(rc, rc);
6319	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
6320	}
6321
6322	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
6323	{
6324	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
6325	if (pVmcsInfo->RealMode.fRealOnV86Active)
6326	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
6327	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
6328	AssertRCReturn(rc, rc);
6329	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
6330	}
6331
6332	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
6333	{
6334	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
6335	if (pVmcsInfo->RealMode.fRealOnV86Active)
6336	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
6337	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
6338	AssertRCReturn(rc, rc);
6339	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
6340	}
6341
6342	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
6343	{
6344	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
6345	if (pVmcsInfo->RealMode.fRealOnV86Active)
6346	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
6347	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
6348	AssertRCReturn(rc, rc);
6349	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
6350	}
6351
6352	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
6353	{
6354	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
6355	if (pVmcsInfo->RealMode.fRealOnV86Active)
6356	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
6357	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
6358	AssertRCReturn(rc, rc);
6359	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
6360	}
6361
6362	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
6363	{
6364	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
6365	if (pVmcsInfo->RealMode.fRealOnV86Active)
6366	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
6367	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
6368	AssertRCReturn(rc, rc);
6369	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
6370	}
6371
6372	#ifdef VBOX_STRICT
6373	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
6374	#endif
6375	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
6376	pCtx->cs.Attr.u));
6377	}
6378
6379	/*
6380	* Guest TR.
6381	*/
6382	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
6383	{
6384	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
6385
6386	/*
6387	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
6388	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
6389	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
6390	*/
6391	uint16_t u16Sel;
6392	uint32_t u32Limit;
6393	uint64_t u64Base;
6394	uint32_t u32AccessRights;
6395	if (!pVmcsInfo->RealMode.fRealOnV86Active)
6396	{
6397	u16Sel = pCtx->tr.Sel;
6398	u32Limit = pCtx->tr.u32Limit;
6399	u64Base = pCtx->tr.u64Base;
6400	u32AccessRights = pCtx->tr.Attr.u;
6401	}
6402	else
6403	{
6404	Assert(!pVmxTransient->fIsNestedGuest);
6405	Assert(pVM->hm.s.vmx.pRealModeTSS);
6406	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
6407
6408	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
6409	RTGCPHYS GCPhys;
6410	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
6411	AssertRCReturn(rc, rc);
6412
6413	X86DESCATTR DescAttr;
6414	DescAttr.u = 0;
6415	DescAttr.n.u1Present = 1;
6416	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
6417
6418	u16Sel = 0;
6419	u32Limit = HM_VTX_TSS_SIZE;
6420	u64Base = GCPhys;
6421	u32AccessRights = DescAttr.u;
6422	}
6423
6424	/* Validate. */
6425	Assert(!(u16Sel & RT_BIT(2)));
6426	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
6427	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
6428	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
6429	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
6430	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
6431	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
6432	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
6433	Assert( (u32Limit & 0xfff) == 0xfff
6434	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
6435	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
6436	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
6437
6438	rc = VMXWriteVmcs16(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
6439	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
6440	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
6441	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
6442	AssertRCReturn(rc, rc);
6443
6444	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
6445	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
6446	}
6447
6448	/*
6449	* Guest GDTR.
6450	*/
6451	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
6452	{
6453	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
6454
6455	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
6456	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
6457	AssertRCReturn(rc, rc);
6458
6459	/* Validate. */
6460	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6461
6462	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
6463	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
6464	}
6465
6466	/*
6467	* Guest LDTR.
6468	*/
6469	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
6470	{
6471	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
6472
6473	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
6474	uint32_t u32Access;
6475	if ( !pVmxTransient->fIsNestedGuest
6476	&& !pCtx->ldtr.Attr.u)
6477	u32Access = X86DESCATTR_UNUSABLE;
6478	else
6479	u32Access = pCtx->ldtr.Attr.u;
6480
6481	rc = VMXWriteVmcs16(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
6482	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
6483	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
6484	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
6485	AssertRCReturn(rc, rc);
6486
6487	/* Validate. */
6488	if (!(u32Access & X86DESCATTR_UNUSABLE))
6489	{
6490	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
6491	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
6492	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
6493	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
6494	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
6495	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
6496	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
6497	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
6498	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
6499	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
6500	}
6501
6502	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
6503	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
6504	}
6505
6506	/*
6507	* Guest IDTR.
6508	*/
6509	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
6510	{
6511	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
6512
6513	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
6514	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
6515	AssertRCReturn(rc, rc);
6516
6517	/* Validate. */
6518	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
6519
6520	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
6521	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
6522	}
6523
6524	return VINF_SUCCESS;
6525	}
6526
6527
6528	/**
6529	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
6530	* areas.
6531	*
6532	* These MSRs will automatically be loaded to the host CPU on every successful
6533	* VM-entry and stored from the host CPU on every successful VM-exit.
6534	*
6535	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
6536	* actual host MSR values are not- updated here for performance reasons. See
6537	* hmR0VmxExportHostMsrs().
6538	*
6539	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
6540	*
6541	* @returns VBox status code.
6542	* @param pVCpu The cross context virtual CPU structure.
6543	* @param pVmxTransient The VMX-transient structure.
6544	*
6545	* @remarks No-long-jump zone!!!
6546	*/
6547	static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6548	{
6549	AssertPtr(pVCpu);
6550	AssertPtr(pVmxTransient);
6551
6552	PVM pVM = pVCpu->CTX_SUFF(pVM);
6553	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6554
6555	/*
6556	* MSRs that we use the auto-load/store MSR area in the VMCS.
6557	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
6558	* The host MSR values are updated when it's safe in hmR0VmxLazySaveHostMsrs().
6559	*
6560	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
6561	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
6562	* emulation, nothing to do here.
6563	*/
6564	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
6565	{
6566	if ( !pVmxTransient->fIsNestedGuest
6567	&& pVM->hm.s.fAllow64BitGuests)
6568	{
6569	#if HC_ARCH_BITS == 32
6570	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_KERNEL_GS_BASE);
6571	Assert(!pVmxTransient->fIsNestedGuest);
6572
6573	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_LSTAR, pCtx->msrLSTAR, true, false);
6574	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_STAR, pCtx->msrSTAR, true, false);
6575	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_SF_MASK, pCtx->msrSFMASK, true, false);
6576	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, true, false);
6577	AssertRCReturn(rc, rc);
6578	#endif
6579	}
6580	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
6581	}
6582
6583	/*
6584	* Guest Sysenter MSRs.
6585	*/
6586	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
6587	{
6588	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
6589
6590	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
6591	{
6592	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
6593	AssertRCReturn(rc, rc);
6594	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
6595	}
6596
6597	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
6598	{
6599	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
6600	AssertRCReturn(rc, rc);
6601	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
6602	}
6603
6604	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
6605	{
6606	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
6607	AssertRCReturn(rc, rc);
6608	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
6609	}
6610	}
6611
6612	/*
6613	* Guest/host EFER MSR.
6614	*/
6615	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
6616	{
6617	/* Whether we are using the VMCS to swap the EFER MSR must have been
6618	determined earlier while exporting VM-entry/VM-exit controls. */
6619	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
6620	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
6621
6622	if (hmR0VmxShouldSwapEferMsr(pVCpu))
6623	{
6624	/*
6625	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
6626	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
6627	*/
6628	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
6629	{
6630	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
6631	AssertRCReturn(rc, rc);
6632	}
6633	else
6634	{
6635	/*
6636	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
6637	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
6638	*/
6639	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, pCtx->msrEFER,
6640	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6641	AssertRCReturn(rc, rc);
6642	}
6643	}
6644	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
6645	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
6646
6647	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
6648	}
6649
6650	/*
6651	* Other MSRs.
6652	* Speculation Control (R/W).
6653	*/
6654	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
6655	{
6656	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
6657	if (pVM->cpum.ro.GuestFeatures.fIbrs)
6658	{
6659	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
6660	false /* fSetReadWrite /, false / fUpdateHostMsr */);
6661	AssertRCReturn(rc, rc);
6662	}
6663	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
6664	}
6665
6666	return VINF_SUCCESS;
6667	}
6668
6669
6670	/**
6671	* Selects up the appropriate function to run guest code.
6672	*
6673	* @returns VBox status code.
6674	* @param pVCpu The cross context virtual CPU structure.
6675	* @param pVmxTransient The VMX-transient structure.
6676	*
6677	* @remarks No-long-jump zone!!!
6678	*/
6679	static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6680	{
6681	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6682	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6683
6684	if (CPUMIsGuestInLongModeEx(pCtx))
6685	{
6686	#ifndef VBOX_WITH_64_BITS_GUESTS
6687	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
6688	#else
6689	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
6690	/* Guest is in long mode, use the 64-bit handler (host is 64-bit). */
6691	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
6692	#endif
6693	}
6694	else
6695	{
6696	/* Guest is not in long mode, use the 32-bit handler. */
6697	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6698	}
6699	Assert(pVmcsInfo->pfnStartVM);
6700	return VINF_SUCCESS;
6701	}
6702
6703
6704	/**
6705	* Wrapper for running the guest code in VT-x.
6706	*
6707	* @returns VBox status code, no informational status codes.
6708	* @param pVCpu The cross context virtual CPU structure.
6709	* @param pVmxTransient The VMX-transient structure.
6710	*
6711	* @remarks No-long-jump zone!!!
6712	*/
6713	DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient)
6714	{
6715	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
6716	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6717	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
6718
6719	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
6720
6721	/*
6722	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
6723	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
6724	* callee-saved and thus the need for this XMM wrapper.
6725	*
6726	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
6727	*/
6728	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6729	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
6730	PVM pVM = pVCpu->CTX_SUFF(pVM);
6731	#ifdef VBOX_WITH_KERNEL_USING_XMM
6732	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu, pVmcsInfo->pfnStartVM);
6733	#else
6734	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu);
6735	#endif
6736	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
6737	return rc;
6738	}
6739
6740
6741	/**
6742	* Reports world-switch error and dumps some useful debug info.
6743	*
6744	* @param pVCpu The cross context virtual CPU structure.
6745	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
6746	* @param pVmxTransient The VMX-transient structure (only
6747	* exitReason updated).
6748	*/
6749	static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
6750	{
6751	Assert(pVCpu);
6752	Assert(pVmxTransient);
6753	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6754
6755	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
6756	switch (rcVMRun)
6757	{
6758	case VERR_VMX_INVALID_VMXON_PTR:
6759	AssertFailed();
6760	break;
6761	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
6762	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
6763	{
6764	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
6765	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
6766	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
6767	AssertRC(rc);
6768
6769	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6770	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
6771	Cannot do it here as we may have been long preempted. */
6772
6773	#ifdef VBOX_STRICT
6774	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6775	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
6776	pVmxTransient->uExitReason));
6777	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
6778	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
6779	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
6780	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
6781	else
6782	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
6783	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
6784	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
6785
6786	static struct
6787	{
6788	/** Name of the field to log. */
6789	const char *pszName;
6790	/** The VMCS field. */
6791	uint32_t uVmcsField;
6792	/** Whether host support of this field needs to be checked. */
6793	bool fCheckSupport;
6794	} const s_aVmcsFields[] =
6795	{
6796	{ "VMX_VMCS32_CTRL_PIN_EXEC", VMX_VMCS32_CTRL_PIN_EXEC, false, },
6797	{ "VMX_VMCS32_CTRL_PROC_EXEC", VMX_VMCS32_CTRL_PROC_EXEC, false, },
6798	{ "VMX_VMCS32_CTRL_PROC_EXEC2", VMX_VMCS32_CTRL_PROC_EXEC2, true, },
6799	{ "VMX_VMCS32_CTRL_ENTRY", VMX_VMCS32_CTRL_ENTRY, false, },
6800	{ "VMX_VMCS32_CTRL_EXIT", VMX_VMCS32_CTRL_EXIT, false, },
6801	{ "VMX_VMCS32_CTRL_CR3_TARGET_COUNT", VMX_VMCS32_CTRL_CR3_TARGET_COUNT, false, },
6802	{ "VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO", VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, false, },
6803	{ "VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE", VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, false, },
6804	{ "VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH", VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, false, },
6805	{ "VMX_VMCS32_CTRL_TPR_THRESHOLD", VMX_VMCS32_CTRL_TPR_THRESHOLD, false, },
6806	{ "VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT", VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, false, },
6807	{ "VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT", VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, false, },
6808	{ "VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT", VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, false, },
6809	{ "VMX_VMCS32_CTRL_EXCEPTION_BITMAP", VMX_VMCS32_CTRL_EXCEPTION_BITMAP, false, },
6810	{ "VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK", VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, false, },
6811	{ "VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH", VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, false, },
6812	{ "VMX_VMCS_CTRL_CR0_MASK", VMX_VMCS_CTRL_CR0_MASK, false, },
6813	{ "VMX_VMCS_CTRL_CR0_READ_SHADOW", VMX_VMCS_CTRL_CR0_READ_SHADOW, false, },
6814	{ "VMX_VMCS_CTRL_CR4_MASK", VMX_VMCS_CTRL_CR4_MASK, false, },
6815	{ "VMX_VMCS_CTRL_CR4_READ_SHADOW", VMX_VMCS_CTRL_CR4_READ_SHADOW, false, },
6816	{ "VMX_VMCS64_CTRL_EPTP_FULL", VMX_VMCS64_CTRL_EPTP_FULL, true, },
6817	{ "VMX_VMCS_GUEST_RIP", VMX_VMCS_GUEST_RIP, false, },
6818	{ "VMX_VMCS_GUEST_RSP", VMX_VMCS_GUEST_RSP, false, },
6819	{ "VMX_VMCS_GUEST_RFLAGS", VMX_VMCS_GUEST_RFLAGS, false, },
6820	{ "VMX_VMCS16_VPID", VMX_VMCS16_VPID, true, },
6821	{ "VMX_VMCS_HOST_CR0", VMX_VMCS_HOST_CR0, false, },
6822	{ "VMX_VMCS_HOST_CR3", VMX_VMCS_HOST_CR3, false, },
6823	{ "VMX_VMCS_HOST_CR4", VMX_VMCS_HOST_CR4, false, },
6824	/* The order of selector fields below are fixed! */
6825	{ "VMX_VMCS16_HOST_ES_SEL", VMX_VMCS16_HOST_ES_SEL, false, },
6826	{ "VMX_VMCS16_HOST_CS_SEL", VMX_VMCS16_HOST_CS_SEL, false, },
6827	{ "VMX_VMCS16_HOST_SS_SEL", VMX_VMCS16_HOST_SS_SEL, false, },
6828	{ "VMX_VMCS16_HOST_DS_SEL", VMX_VMCS16_HOST_DS_SEL, false, },
6829	{ "VMX_VMCS16_HOST_FS_SEL", VMX_VMCS16_HOST_FS_SEL, false, },
6830	{ "VMX_VMCS16_HOST_GS_SEL", VMX_VMCS16_HOST_GS_SEL, false, },
6831	{ "VMX_VMCS16_HOST_TR_SEL", VMX_VMCS16_HOST_TR_SEL, false, },
6832	/* End of ordered selector fields. */
6833	{ "VMX_VMCS_HOST_TR_BASE", VMX_VMCS_HOST_TR_BASE, false, },
6834	{ "VMX_VMCS_HOST_GDTR_BASE", VMX_VMCS_HOST_GDTR_BASE, false, },
6835	{ "VMX_VMCS_HOST_IDTR_BASE", VMX_VMCS_HOST_IDTR_BASE, false, },
6836	{ "VMX_VMCS32_HOST_SYSENTER_CS", VMX_VMCS32_HOST_SYSENTER_CS, false, },
6837	{ "VMX_VMCS_HOST_SYSENTER_EIP", VMX_VMCS_HOST_SYSENTER_EIP, false, },
6838	{ "VMX_VMCS_HOST_SYSENTER_ESP", VMX_VMCS_HOST_SYSENTER_ESP, false, },
6839	{ "VMX_VMCS_HOST_RSP", VMX_VMCS_HOST_RSP, false, },
6840	{ "VMX_VMCS_HOST_RIP", VMX_VMCS_HOST_RIP, false, }
6841	};
6842
6843	RTGDTR HostGdtr;
6844	ASMGetGDTR(&HostGdtr);
6845
6846	uint32_t const cVmcsFields = RT_ELEMENTS(s_aVmcsFields);
6847	for (uint32_t i = 0; i < cVmcsFields; i++)
6848	{
6849	uint32_t const uVmcsField = s_aVmcsFields[i].uVmcsField;
6850
6851	bool fSupported;
6852	if (!s_aVmcsFields[i].fCheckSupport)
6853	fSupported = true;
6854	else
6855	{
6856	PVM pVM = pVCpu->CTX_SUFF(pVM);
6857	switch (uVmcsField)
6858	{
6859	case VMX_VMCS64_CTRL_EPTP_FULL: fSupported = pVM->hm.s.fNestedPaging; break;
6860	case VMX_VMCS16_VPID: fSupported = pVM->hm.s.vmx.fVpid; break;
6861	case VMX_VMCS32_CTRL_PROC_EXEC2:
6862	fSupported = RT_BOOL(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
6863	break;
6864	default:
6865	AssertMsgFailedReturnVoid(("Failed to provide VMCS field support for %#RX32\n", uVmcsField));
6866	}
6867	}
6868
6869	if (fSupported)
6870	{
6871	const char *pszName = s_aVmcsFields[i].pszName;
6872	uint8_t const uWidth = RT_BF_GET(uVmcsField, VMX_BF_VMCSFIELD_WIDTH);
6873	switch (uWidth)
6874	{
6875	case VMX_VMCSFIELD_WIDTH_16BIT:
6876	{
6877	uint16_t u16Val;
6878	rc = VMXReadVmcs16(uVmcsField, &u16Val);
6879	AssertRC(rc);
6880	Log4(("%-40s = %#RX16\n", pszName, u16Val));
6881
6882	if ( uVmcsField >= VMX_VMCS16_HOST_ES_SEL
6883	&& uVmcsField <= VMX_VMCS16_HOST_TR_SEL)
6884	{
6885	if (u16Val < HostGdtr.cbGdt)
6886	{
6887	/* Order of selectors in s_apszSel is fixed and matches the order in s_aVmcsFields. */
6888	static const char * const s_apszSel[] = { "Host ES", "Host CS", "Host SS", "Host DS",
6889	"Host FS", "Host GS", "Host TR" };
6890	uint8_t const idxSel = RT_BF_GET(uVmcsField, VMX_BF_VMCSFIELD_INDEX);
6891	Assert(idxSel < RT_ELEMENTS(s_apszSel));
6892	PCX86DESCHC pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u16Val & X86_SEL_MASK));
6893	hmR0DumpDescriptor(pDesc, u16Val, s_apszSel[idxSel]);
6894	}
6895	else
6896	Log4((" Selector value exceeds GDT limit!\n"));
6897	}
6898	break;
6899	}
6900
6901	case VMX_VMCSFIELD_WIDTH_32BIT:
6902	{
6903	uint32_t u32Val;
6904	rc = VMXReadVmcs32(uVmcsField, &u32Val);
6905	AssertRC(rc);
6906	Log4(("%-40s = %#RX32\n", pszName, u32Val));
6907	break;
6908	}
6909
6910	case VMX_VMCSFIELD_WIDTH_64BIT:
6911	case VMX_VMCSFIELD_WIDTH_NATURAL:
6912	{
6913	uint64_t u64Val;
6914	rc = VMXReadVmcs64(uVmcsField, &u64Val);
6915	AssertRC(rc);
6916	Log4(("%-40s = %#RX64\n", pszName, u64Val));
6917	break;
6918	}
6919	}
6920	}
6921	}
6922
6923	# if HC_ARCH_BITS == 64
6924	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
6925	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
6926	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
6927	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
6928	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
6929	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
6930	# endif
6931	#endif /* VBOX_STRICT */
6932	break;
6933	}
6934
6935	default:
6936	/* Impossible */
6937	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
6938	break;
6939	}
6940	}
6941
6942
6943	/**
6944	* Sets up the usage of TSC-offsetting and updates the VMCS.
6945	*
6946	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
6947	* VMX-preemption timer.
6948	*
6949	* @returns VBox status code.
6950	* @param pVCpu The cross context virtual CPU structure.
6951	* @param pVmxTransient The VMX-transient structure.
6952	*
6953	* @remarks No-long-jump zone!!!
6954	*/
6955	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6956	{
6957	bool fOffsettedTsc;
6958	bool fParavirtTsc;
6959	uint64_t uTscOffset;
6960	PVM pVM = pVCpu->CTX_SUFF(pVM);
6961	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6962
6963	if (pVM->hm.s.vmx.fUsePreemptTimer)
6964	{
6965	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
6966
6967	/* Make sure the returned values have sane upper and lower boundaries. */
6968	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
6969	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
6970	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
6971	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
6972
6973	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
6974	* preemption timers here. We probably need to clamp the preemption timer,
6975	* after converting the timer value to the host. */
6976	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
6977	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
6978	AssertRC(rc);
6979	}
6980	else
6981	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
6982
6983	if (fParavirtTsc)
6984	{
6985	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
6986	information before every VM-entry, hence disable it for performance sake. */
6987	#if 0
6988	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
6989	AssertRC(rc);
6990	#endif
6991	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
6992	}
6993
6994	if ( fOffsettedTsc
6995	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
6996	{
6997	if (pVmxTransient->fIsNestedGuest)
6998	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
6999	hmR0VmxSetTscOffsetVmcs(pVCpu, pVmcsInfo, uTscOffset);
7000	hmR0VmxRemoveProcCtlsVmcs(pVCpu, pVmxTransient, VMX_PROC_CTLS_RDTSC_EXIT);
7001	}
7002	else
7003	{
7004	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
7005	hmR0VmxSetProcCtlsVmcs(pVmxTransient, VMX_PROC_CTLS_RDTSC_EXIT);
7006	}
7007	}
7008
7009
7010	/**
7011	* Gets the IEM exception flags for the specified vector and IDT vectoring /
7012	* VM-exit interruption info type.
7013	*
7014	* @returns The IEM exception flags.
7015	* @param uVector The event vector.
7016	* @param uVmxEventType The VMX event type.
7017	*
7018	* @remarks This function currently only constructs flags required for
7019	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
7020	* and CR2 aspects of an exception are not included).
7021	*/
7022	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
7023	{
7024	uint32_t fIemXcptFlags;
7025	switch (uVmxEventType)
7026	{
7027	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7028	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7029	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
7030	break;
7031
7032	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7033	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
7034	break;
7035
7036	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7037	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
7038	break;
7039
7040	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
7041	{
7042	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
7043	if (uVector == X86_XCPT_BP)
7044	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
7045	else if (uVector == X86_XCPT_OF)
7046	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
7047	else
7048	{
7049	fIemXcptFlags = 0;
7050	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
7051	}
7052	break;
7053	}
7054
7055	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7056	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
7057	break;
7058
7059	default:
7060	fIemXcptFlags = 0;
7061	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
7062	break;
7063	}
7064	return fIemXcptFlags;
7065	}
7066
7067
7068	/**
7069	* Sets an event as a pending event to be injected into the guest.
7070	*
7071	* @param pVCpu The cross context virtual CPU structure.
7072	* @param u32IntInfo The VM-entry interruption-information field.
7073	* @param cbInstr The VM-entry instruction length in bytes (for software
7074	* interrupts, exceptions and privileged software
7075	* exceptions).
7076	* @param u32ErrCode The VM-entry exception error code.
7077	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
7078	* page-fault.
7079	*/
7080	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7081	RTGCUINTPTR GCPtrFaultAddress)
7082	{
7083	Assert(!pVCpu->hm.s.Event.fPending);
7084	pVCpu->hm.s.Event.fPending = true;
7085	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
7086	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
7087	pVCpu->hm.s.Event.cbInstr = cbInstr;
7088	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
7089	}
7090
7091
7092	/**
7093	* Sets an external interrupt as pending-for-injection into the VM.
7094	*
7095	* @param pVCpu The cross context virtual CPU structure.
7096	* @param u8Interrupt The external interrupt vector.
7097	*/
7098	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPU pVCpu, uint8_t u8Interrupt)
7099	{
7100	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
7101	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
7102	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7103	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7104	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7105	}
7106
7107
7108	/**
7109	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
7110	*
7111	* @param pVCpu The cross context virtual CPU structure.
7112	*/
7113	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPU pVCpu)
7114	{
7115	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
7116	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
7117	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7118	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7119	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7120	}
7121
7122
7123	/**
7124	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
7125	*
7126	* @param pVCpu The cross context virtual CPU structure.
7127	*/
7128	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
7129	{
7130	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
7131	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7132	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7133	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7134	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7135	}
7136
7137
7138	/**
7139	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7140	*
7141	* @param pVCpu The cross context virtual CPU structure.
7142	*/
7143	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
7144	{
7145	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
7146	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7147	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7148	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7149	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7150	}
7151
7152
7153	/**
7154	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
7155	*
7156	* @param pVCpu The cross context virtual CPU structure.
7157	*/
7158	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
7159	{
7160	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
7161	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7162	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
7163	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7164	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7165	}
7166
7167
7168	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7169	/**
7170	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
7171	*
7172	* @param pVCpu The cross context virtual CPU structure.
7173	* @param u32ErrCode The error code for the general-protection exception.
7174	*/
7175	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
7176	{
7177	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
7178	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7179	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7180	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7181	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
7182	}
7183
7184
7185	/**
7186	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
7187	*
7188	* @param pVCpu The cross context virtual CPU structure.
7189	* @param u32ErrCode The error code for the stack exception.
7190	*/
7191	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
7192	{
7193	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
7194	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
7195	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
7196	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
7197	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
7198	}
7199	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7200
7201
7202	/**
7203	* Fixes up attributes for the specified segment register.
7204	*
7205	* @param pVCpu The cross context virtual CPU structure.
7206	* @param pSelReg The segment register that needs fixing.
7207	* @param idxSel The VMCS field for the corresponding segment register.
7208	*/
7209	static void hmR0VmxFixUnusableSegRegAttr(PVMCPU pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7210	{
7211	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7212
7213	/*
7214	* If VT-x marks the segment as unusable, most other bits remain undefined:
7215	* - For CS the L, D and G bits have meaning.
7216	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7217	* - For the remaining data segments no bits are defined.
7218	*
7219	* The present bit and the unusable bit has been observed to be set at the
7220	* same time (the selector was supposed to be invalid as we started executing
7221	* a V8086 interrupt in ring-0).
7222	*
7223	* What should be important for the rest of the VBox code, is that the P bit is
7224	* cleared. Some of the other VBox code recognizes the unusable bit, but
7225	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7226	* safe side here, we'll strip off P and other bits we don't care about. If
7227	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7228	*
7229	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7230	*/
7231	#ifdef VBOX_STRICT
7232	uint32_t const uAttr = pSelReg->Attr.u;
7233	#endif
7234
7235	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7236	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7237	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7238
7239	#ifdef VBOX_STRICT
7240	VMMRZCallRing3Disable(pVCpu);
7241	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7242	# ifdef DEBUG_bird
7243	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7244	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7245	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7246	# endif
7247	VMMRZCallRing3Enable(pVCpu);
7248	NOREF(uAttr);
7249	#endif
7250	RT_NOREF2(pVCpu, idxSel);
7251	}
7252
7253
7254	/**
7255	* Imports a guest segment register from the current VMCS into the guest-CPU
7256	* context.
7257	*
7258	* @returns VBox status code.
7259	* @param pVCpu The cross context virtual CPU structure.
7260	* @param iSegReg The segment register number (X86_SREG_XXX).
7261	*
7262	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7263	* do not log!
7264	*/
7265	static int hmR0VmxImportGuestSegReg(PVMCPU pVCpu, uint8_t iSegReg)
7266	{
7267	Assert(iSegReg < X86_SREG_COUNT);
7268
7269	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7270	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7271	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7272	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7273
7274	uint16_t u16Sel;
7275	uint64_t u64Base;
7276	uint32_t u32Limit, u32Attr;
7277	int rc = VMXReadVmcs16(idxSel, &u16Sel);
7278	rc \|= VMXReadVmcs32(idxLimit, &u32Limit);
7279	rc \|= VMXReadVmcs32(idxAttr, &u32Attr);
7280	rc \|= VMXReadVmcsGstN(idxBase, &u64Base);
7281	if (RT_SUCCESS(rc))
7282	{
7283	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7284	pSelReg->Sel = u16Sel;
7285	pSelReg->ValidSel = u16Sel;
7286	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7287	pSelReg->u32Limit = u32Limit;
7288	pSelReg->u64Base = u64Base;
7289	pSelReg->Attr.u = u32Attr;
7290	if (u32Attr & X86DESCATTR_UNUSABLE)
7291	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7292	}
7293	return rc;
7294	}
7295
7296
7297	/**
7298	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7299	*
7300	* @returns VBox status code.
7301	* @param pVCpu The cross context virtual CPU structure.
7302	*
7303	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7304	* do not log!
7305	*/
7306	static int hmR0VmxImportGuestLdtr(PVMCPU pVCpu)
7307	{
7308	uint16_t u16Sel;
7309	uint64_t u64Base;
7310	uint32_t u32Limit, u32Attr;
7311	int rc = VMXReadVmcs16(VMX_VMCS16_GUEST_LDTR_SEL, &u16Sel);
7312	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit);
7313	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr);
7314	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, &u64Base);
7315	if (RT_SUCCESS(rc))
7316	{
7317	pVCpu->cpum.GstCtx.ldtr.Sel = u16Sel;
7318	pVCpu->cpum.GstCtx.ldtr.ValidSel = u16Sel;
7319	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7320	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7321	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7322	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7323	if (u32Attr & X86DESCATTR_UNUSABLE)
7324	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7325	}
7326	return rc;
7327	}
7328
7329
7330	/**
7331	* Imports the guest TR from the current VMCS into the guest-CPU context.
7332	*
7333	* @returns VBox status code.
7334	* @param pVCpu The cross context virtual CPU structure.
7335	*
7336	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7337	* do not log!
7338	*/
7339	static int hmR0VmxImportGuestTr(PVMCPU pVCpu)
7340	{
7341	uint16_t u16Sel;
7342	uint64_t u64Base;
7343	uint32_t u32Limit, u32Attr;
7344	int rc = VMXReadVmcs16(VMX_VMCS16_GUEST_TR_SEL, &u16Sel);
7345	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit);
7346	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr);
7347	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_TR_BASE, &u64Base);
7348	AssertRCReturn(rc, rc);
7349
7350	pVCpu->cpum.GstCtx.tr.Sel = u16Sel;
7351	pVCpu->cpum.GstCtx.tr.ValidSel = u16Sel;
7352	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7353	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7354	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7355	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7356	/* TR is the only selector that can never be unusable. */
7357	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7358	return VINF_SUCCESS;
7359	}
7360
7361
7362	/**
7363	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7364	*
7365	* @returns VBox status code.
7366	* @param pVCpu The cross context virtual CPU structure.
7367	*
7368	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7369	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7370	* instead!!!
7371	*/
7372	static int hmR0VmxImportGuestRip(PVMCPU pVCpu)
7373	{
7374	uint64_t u64Val;
7375	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7376	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7377	{
7378	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
7379	if (RT_SUCCESS(rc))
7380	{
7381	pCtx->rip = u64Val;
7382	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7383	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7384	}
7385	return rc;
7386	}
7387	return VINF_SUCCESS;
7388	}
7389
7390
7391	/**
7392	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7393	*
7394	* @returns VBox status code.
7395	* @param pVCpu The cross context virtual CPU structure.
7396	* @param pVmcsInfo The VMCS info. object.
7397	*
7398	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7399	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7400	* instead!!!
7401	*/
7402	static int hmR0VmxImportGuestRFlags(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7403	{
7404	uint32_t u32Val;
7405	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7406	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7407	{
7408	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
7409	if (RT_SUCCESS(rc))
7410	{
7411	pCtx->eflags.u32 = u32Val;
7412
7413	/* Restore eflags for real-on-v86-mode hack. */
7414	if (pVmcsInfo->RealMode.fRealOnV86Active)
7415	{
7416	pCtx->eflags.Bits.u1VM = 0;
7417	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7418	}
7419	}
7420	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7421	return rc;
7422	}
7423	return VINF_SUCCESS;
7424	}
7425
7426
7427	/**
7428	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7429	* context.
7430	*
7431	* @returns VBox status code.
7432	* @param pVCpu The cross context virtual CPU structure.
7433	* @param pVmcsInfo The VMCS info. object.
7434	*
7435	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7436	* do not log!
7437	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7438	* instead!!!
7439	*/
7440	static int hmR0VmxImportGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7441	{
7442	uint32_t u32Val;
7443	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
7444	if (RT_SUCCESS(rc))
7445	{
7446	if (!u32Val)
7447	{
7448	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7449	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7450
7451	CPUMSetGuestNmiBlocking(pVCpu, false);
7452	}
7453	else
7454	{
7455	/*
7456	* We must import RIP here to set our EM interrupt-inhibited state.
7457	* We also import RFLAGS as our code that evaluates pending interrupts
7458	* before VM-entry requires it.
7459	*/
7460	rc = hmR0VmxImportGuestRip(pVCpu);
7461	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7462	if (RT_SUCCESS(rc))
7463	{
7464	if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS \| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7465	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7466	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7467	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7468
7469	bool const fNmiBlocking = RT_BOOL(u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7470	CPUMSetGuestNmiBlocking(pVCpu, fNmiBlocking);
7471	}
7472	}
7473	}
7474	return rc;
7475	}
7476
7477
7478	/**
7479	* Worker for VMXR0ImportStateOnDemand.
7480	*
7481	* @returns VBox status code.
7482	* @param pVCpu The cross context virtual CPU structure.
7483	* @param pVmcsInfo The VMCS info. object.
7484	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7485	*/
7486	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
7487	{
7488	#define VMXLOCAL_BREAK_RC(a_rc) \
7489	if (RT_SUCCESS(a_rc)) \
7490	{ } \
7491	else \
7492	break
7493
7494	int rc = VINF_SUCCESS;
7495	PVM pVM = pVCpu->CTX_SUFF(pVM);
7496	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7497	uint64_t u64Val;
7498	uint32_t u32Val;
7499
7500	/*
7501	* Note! This is hack to workaround a mysterious BSOD observed with release builds
7502	* on Windows 10 64-bit hosts. Profile and debug builds are not affected and
7503	* neither are other host platforms.
7504	*
7505	* Committing this temporarily as it prevents BSOD.
7506	*
7507	* Update: This is very likely a compiler optimization bug, see @bugref{9180}.
7508	*/
7509	#ifdef RT_OS_WINDOWS
7510	if (pVM == 0 \|\| pVM == (void *)(uintptr_t)-1)
7511	return VERR_HM_IPE_1;
7512	#endif
7513
7514	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
7515
7516	/*
7517	* We disable interrupts to make the updating of the state and in particular
7518	* the fExtrn modification atomic wrt to preemption hooks.
7519	*/
7520	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
7521
7522	fWhat &= pCtx->fExtrn;
7523	if (fWhat)
7524	{
7525	do
7526	{
7527	if (fWhat & CPUMCTX_EXTRN_RIP)
7528	{
7529	rc = hmR0VmxImportGuestRip(pVCpu);
7530	VMXLOCAL_BREAK_RC(rc);
7531	}
7532
7533	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
7534	{
7535	rc = hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7536	VMXLOCAL_BREAK_RC(rc);
7537	}
7538
7539	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
7540	{
7541	rc = hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
7542	VMXLOCAL_BREAK_RC(rc);
7543	}
7544
7545	if (fWhat & CPUMCTX_EXTRN_RSP)
7546	{
7547	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
7548	VMXLOCAL_BREAK_RC(rc);
7549	pCtx->rsp = u64Val;
7550	}
7551
7552	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
7553	{
7554	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
7555	if (fWhat & CPUMCTX_EXTRN_CS)
7556	{
7557	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
7558	rc \|= hmR0VmxImportGuestRip(pVCpu);
7559	if (fRealOnV86Active)
7560	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
7561	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
7562	}
7563	if (fWhat & CPUMCTX_EXTRN_SS)
7564	{
7565	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
7566	if (fRealOnV86Active)
7567	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
7568	}
7569	if (fWhat & CPUMCTX_EXTRN_DS)
7570	{
7571	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
7572	if (fRealOnV86Active)
7573	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
7574	}
7575	if (fWhat & CPUMCTX_EXTRN_ES)
7576	{
7577	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
7578	if (fRealOnV86Active)
7579	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
7580	}
7581	if (fWhat & CPUMCTX_EXTRN_FS)
7582	{
7583	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
7584	if (fRealOnV86Active)
7585	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
7586	}
7587	if (fWhat & CPUMCTX_EXTRN_GS)
7588	{
7589	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
7590	if (fRealOnV86Active)
7591	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
7592	}
7593	VMXLOCAL_BREAK_RC(rc);
7594	}
7595
7596	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
7597	{
7598	if (fWhat & CPUMCTX_EXTRN_LDTR)
7599	rc \|= hmR0VmxImportGuestLdtr(pVCpu);
7600
7601	if (fWhat & CPUMCTX_EXTRN_GDTR)
7602	{
7603	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
7604	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
7605	pCtx->gdtr.pGdt = u64Val;
7606	pCtx->gdtr.cbGdt = u32Val;
7607	}
7608
7609	/* Guest IDTR. */
7610	if (fWhat & CPUMCTX_EXTRN_IDTR)
7611	{
7612	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
7613	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
7614	pCtx->idtr.pIdt = u64Val;
7615	pCtx->idtr.cbIdt = u32Val;
7616	}
7617
7618	/* Guest TR. */
7619	if (fWhat & CPUMCTX_EXTRN_TR)
7620	{
7621	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
7622	don't need to import that one. */
7623	if (!pVmcsInfo->RealMode.fRealOnV86Active)
7624	rc \|= hmR0VmxImportGuestTr(pVCpu);
7625	}
7626	VMXLOCAL_BREAK_RC(rc);
7627	}
7628
7629	if (fWhat & CPUMCTX_EXTRN_DR7)
7630	{
7631	if (!pVCpu->hm.s.fUsingHyperDR7)
7632	{
7633	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
7634	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
7635	VMXLOCAL_BREAK_RC(rc);
7636	pCtx->dr[7] = u32Val;
7637	}
7638	}
7639
7640	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
7641	{
7642	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
7643	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
7644	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
7645	pCtx->SysEnter.cs = u32Val;
7646	VMXLOCAL_BREAK_RC(rc);
7647	}
7648
7649	#if HC_ARCH_BITS == 64
7650	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
7651	{
7652	if ( pVM->hm.s.fAllow64BitGuests
7653	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7654	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
7655	}
7656
7657	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
7658	{
7659	if ( pVM->hm.s.fAllow64BitGuests
7660	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7661	{
7662	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
7663	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
7664	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
7665	}
7666	}
7667	#endif
7668
7669	if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
7670	#if HC_ARCH_BITS == 32
7671	\|\| (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS))
7672	#endif
7673	)
7674	{
7675	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
7676	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
7677	Assert(pMsrs);
7678	Assert(cMsrs <= VMX_MISC_MAX_MSRS(pVM->hm.s.vmx.Msrs.u64Misc));
7679	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
7680	for (uint32_t i = 0; i < cMsrs; i++)
7681	{
7682	uint32_t const idMsr = pMsrs[i].u32Msr;
7683	switch (idMsr)
7684	{
7685	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
7686	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
7687	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
7688	#if HC_ARCH_BITS == 32
7689	case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsrs[i].u64Value; break;
7690	case MSR_K6_STAR: pCtx->msrSTAR = pMsrs[i].u64Value; break;
7691	case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsrs[i].u64Value; break;
7692	case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsrs[i].u64Value; break;
7693	#endif
7694	default:
7695	{
7696	pCtx->fExtrn = 0;
7697	pVCpu->hm.s.u32HMError = pMsrs->u32Msr;
7698	ASMSetFlags(fEFlags);
7699	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
7700	return VERR_HM_UNEXPECTED_LD_ST_MSR;
7701	}
7702	}
7703	}
7704	}
7705
7706	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
7707	{
7708	uint64_t u64Shadow;
7709	if (fWhat & CPUMCTX_EXTRN_CR0)
7710	{
7711	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7712	* remove when we drop 32-bit host w/ 64-bit host support, see
7713	* @bugref{9180#c39}. */
7714	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
7715	#if HC_ARCH_BITS == 32
7716	uint32_t u32Shadow;
7717	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
7718	u64Shadow = u32Shadow;
7719	#else
7720	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow);
7721	#endif
7722	VMXLOCAL_BREAK_RC(rc);
7723	u64Val = u32Val;
7724	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
7725	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7726	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7727	/*
7728	* Reapply the nested-guest's CR0 fixed bits that might have been altered while
7729	* exporting the nested-guest CR0 for executing using hardware-assisted VMX.
7730	*/
7731	if (CPUMIsGuestInVmxNonRootMode(pCtx))
7732	{
7733	u64Val \|= pCtx->hwvirt.vmx.Msrs.u64Cr0Fixed0;
7734	u64Val &= pCtx->hwvirt.vmx.Msrs.u64Cr0Fixed1;
7735	}
7736	#endif
7737	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
7738	CPUMSetGuestCR0(pVCpu, u64Val);
7739	VMMRZCallRing3Enable(pVCpu);
7740	}
7741
7742	if (fWhat & CPUMCTX_EXTRN_CR4)
7743	{
7744	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7745	* remove when we drop 32-bit host w/ 64-bit host support, see
7746	* @bugref{9180#c39}. */
7747	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
7748	#if HC_ARCH_BITS == 32
7749	uint32_t u32Shadow;
7750	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
7751	u64Shadow = u32Shadow;
7752	#else
7753	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow);
7754	#endif
7755	VMXLOCAL_BREAK_RC(rc);
7756	u64Val = u32Val;
7757	u64Val = (u64Val & ~pVmcsInfo->u64Cr4Mask)
7758	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7759	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7760	/*
7761	* Reapply the nested-guest's CR4 fixed bits that might have been altered while
7762	* exporting the nested-guest CR4 for executing using hardware-assisted VMX.
7763	*/
7764	if (CPUMIsGuestInVmxNonRootMode(pCtx))
7765	{
7766	u64Val \|= pCtx->hwvirt.vmx.Msrs.u64Cr4Fixed0;
7767	u64Val &= pCtx->hwvirt.vmx.Msrs.u64Cr4Fixed1;
7768	}
7769	#endif
7770	pCtx->cr4 = u64Val;
7771	}
7772
7773	if (fWhat & CPUMCTX_EXTRN_CR3)
7774	{
7775	/* CR0.PG bit changes are always intercepted, so it's up to date. */
7776	if ( pVM->hm.s.vmx.fUnrestrictedGuest
7777	\|\| ( pVM->hm.s.fNestedPaging
7778	&& CPUMIsGuestPagingEnabledEx(pCtx)))
7779	{
7780	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
7781	VMXLOCAL_BREAK_RC(rc);
7782	if (pCtx->cr3 != u64Val)
7783	{
7784	pCtx->cr3 = u64Val;
7785	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
7786	}
7787
7788	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
7789	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
7790	if (CPUMIsGuestInPAEModeEx(pCtx))
7791	{
7792	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
7793	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
7794	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
7795	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
7796	VMXLOCAL_BREAK_RC(rc);
7797	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
7798	}
7799	}
7800	}
7801	}
7802
7803	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7804	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
7805	{
7806	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
7807	&& !CPUMIsGuestInVmxNonRootMode(pCtx))
7808	{
7809	Assert(CPUMIsGuestInVmxRootMode(pCtx));
7810	rc = hmR0VmxCopyShadowToNstGstVmcs(pVCpu, pVmcsInfo);
7811	VMXLOCAL_BREAK_RC(rc);
7812	}
7813	}
7814	#endif
7815	} while (0);
7816
7817	if (RT_SUCCESS(rc))
7818	{
7819	/* Update fExtrn. */
7820	pCtx->fExtrn &= ~fWhat;
7821
7822	/* If everything has been imported, clear the HM keeper bit. */
7823	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
7824	{
7825	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
7826	Assert(!pCtx->fExtrn);
7827	}
7828	}
7829	}
7830	else
7831	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
7832
7833	ASMSetFlags(fEFlags);
7834
7835	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
7836
7837	if (RT_SUCCESS(rc))
7838	{ /* likely */ }
7839	else
7840	return rc;
7841
7842	/*
7843	* Honor any pending CR3 updates.
7844	*
7845	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
7846	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
7847	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
7848	*
7849	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
7850	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
7851	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
7852	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
7853	*
7854	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
7855	*/
7856	if (VMMRZCallRing3IsEnabled(pVCpu))
7857	{
7858	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7859	{
7860	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
7861	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
7862	}
7863
7864	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7865	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7866
7867	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7868	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7869	}
7870
7871	return VINF_SUCCESS;
7872	#undef VMXLOCAL_BREAK_RC
7873	}
7874
7875
7876	/**
7877	* Saves the guest state from the VMCS into the guest-CPU context.
7878	*
7879	* @returns VBox status code.
7880	* @param pVCpu The cross context virtual CPU structure.
7881	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7882	*/
7883	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
7884	{
7885	AssertPtr(pVCpu);
7886	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7887	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
7888	}
7889
7890
7891	/**
7892	* Check per-VM and per-VCPU force flag actions that require us to go back to
7893	* ring-3 for one reason or another.
7894	*
7895	* @returns Strict VBox status code (i.e. informational status codes too)
7896	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7897	* ring-3.
7898	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7899	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7900	* interrupts)
7901	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7902	* all EMTs to be in ring-3.
7903	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7904	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7905	* to the EM loop.
7906	*
7907	* @param pVCpu The cross context virtual CPU structure.
7908	* @param fStepping Whether we are single-stepping the guest using the
7909	* hypervisor debugger.
7910	*
7911	* @remarks This might cause nested-guest VM-exits, caller must check if the guest
7912	* is no longer in VMX non-root mode.
7913	*/
7914	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
7915	{
7916	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7917
7918	/*
7919	* Update pending interrupts into the APIC's IRR.
7920	*/
7921	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7922	APICUpdatePendingInterrupts(pVCpu);
7923
7924	/*
7925	* Anything pending? Should be more likely than not if we're doing a good job.
7926	*/
7927	PVM pVM = pVCpu->CTX_SUFF(pVM);
7928	if ( !fStepping
7929	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7930	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7931	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7932	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7933	return VINF_SUCCESS;
7934
7935	/* Pending PGM C3 sync. */
7936	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7937	{
7938	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7939	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
7940	VBOXSTRICTRC rcStrict = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
7941	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7942	if (rcStrict != VINF_SUCCESS)
7943	{
7944	AssertRC(VBOXSTRICTRC_VAL(rcStrict));
7945	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
7946	return rcStrict;
7947	}
7948	}
7949
7950	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7951	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
7952	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7953	{
7954	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7955	int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
7956	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
7957	return rc;
7958	}
7959
7960	/* Pending VM request packets, such as hardware interrupts. */
7961	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
7962	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
7963	{
7964	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchVmReq);
7965	Log4Func(("Pending VM request forcing us back to ring-3\n"));
7966	return VINF_EM_PENDING_REQUEST;
7967	}
7968
7969	/* Pending PGM pool flushes. */
7970	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7971	{
7972	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPgmPoolFlush);
7973	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
7974	return VINF_PGM_POOL_FLUSH_PENDING;
7975	}
7976
7977	/* Pending DMA requests. */
7978	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
7979	{
7980	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchDma);
7981	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
7982	return VINF_EM_RAW_TO_R3;
7983	}
7984
7985	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
7986	/* Pending nested-guest APIC-write (has highest priority among nested-guest FFs). */
7987	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
7988	{
7989	Log4Func(("Pending nested-guest APIC-write\n"));
7990	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitApicWrite(pVCpu);
7991	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
7992	return rcStrict;
7993	}
7994	/** @todo VMCPU_FF_VMX_MTF, VMCPU_FF_VMX_PREEMPT_TIMER */
7995	#endif
7996
7997	return VINF_SUCCESS;
7998	}
7999
8000
8001	/**
8002	* Converts any TRPM trap into a pending HM event. This is typically used when
8003	* entering from ring-3 (not longjmp returns).
8004	*
8005	* @param pVCpu The cross context virtual CPU structure.
8006	*/
8007	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
8008	{
8009	Assert(TRPMHasTrap(pVCpu));
8010	Assert(!pVCpu->hm.s.Event.fPending);
8011
8012	uint8_t uVector;
8013	TRPMEVENT enmTrpmEvent;
8014	RTGCUINT uErrCode;
8015	RTGCUINTPTR GCPtrFaultAddress;
8016	uint8_t cbInstr;
8017
8018	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
8019	AssertRC(rc);
8020
8021	uint32_t u32IntInfo;
8022	u32IntInfo = uVector \| VMX_IDT_VECTORING_INFO_VALID;
8023	u32IntInfo \|= HMTrpmEventTypeToVmxEventType(uVector, enmTrpmEvent);
8024
8025	rc = TRPMResetTrap(pVCpu);
8026	AssertRC(rc);
8027	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
8028	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
8029
8030	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
8031	}
8032
8033
8034	/**
8035	* Converts the pending HM event into a TRPM trap.
8036	*
8037	* @param pVCpu The cross context virtual CPU structure.
8038	*/
8039	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
8040	{
8041	Assert(pVCpu->hm.s.Event.fPending);
8042
8043	/* If a trap was already pending, we did something wrong! */
8044	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
8045
8046	uint32_t const u32IntInfo = pVCpu->hm.s.Event.u64IntInfo;
8047	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(u32IntInfo);
8048	TRPMEVENT const enmTrapType = HMVmxEventTypeToTrpmEventType(u32IntInfo);
8049
8050	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
8051
8052	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
8053	AssertRC(rc);
8054
8055	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8056	TRPMSetErrorCode(pVCpu, pVCpu->hm.s.Event.u32ErrCode);
8057
8058	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(u32IntInfo))
8059	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
8060	else if (VMX_IDT_VECTORING_INFO_TYPE(u32IntInfo) == VMX_IDT_VECTORING_INFO_TYPE_SW_INT)
8061	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
8062
8063	/* We're now done converting the pending event. */
8064	pVCpu->hm.s.Event.fPending = false;
8065	}
8066
8067
8068	/**
8069	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
8070	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
8071	*
8072	* @param pVCpu The cross context virtual CPU structure.
8073	* @param pVmcsInfo The VMCS info. object.
8074	*/
8075	static void hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8076	{
8077	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8078	{
8079	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
8080	{
8081	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
8082	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8083	AssertRC(rc);
8084	}
8085	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
8086	}
8087
8088
8089	/**
8090	* Clears the interrupt-window exiting control in the VMCS.
8091	*
8092	* @param pVmcsInfo The VMCS info. object.
8093	*/
8094	DECLINLINE(int) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8095	{
8096	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8097	{
8098	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
8099	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8100	}
8101	return VINF_SUCCESS;
8102	}
8103
8104
8105	/**
8106	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
8107	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
8108	*
8109	* @param pVCpu The cross context virtual CPU structure.
8110	* @param pVmcsInfo The VMCS info. object.
8111	*/
8112	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
8113	{
8114	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8115	{
8116	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
8117	{
8118	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8119	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8120	AssertRC(rc);
8121	Log4Func(("Setup NMI-window exiting\n"));
8122	}
8123	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
8124	}
8125
8126
8127	/**
8128	* Clears the NMI-window exiting control in the VMCS.
8129	*
8130	* @param pVmcsInfo The VMCS info. object.
8131	*/
8132	DECLINLINE(int) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
8133	{
8134	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8135	{
8136	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
8137	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8138	}
8139	return VINF_SUCCESS;
8140	}
8141
8142
8143	/**
8144	* Does the necessary state syncing before returning to ring-3 for any reason
8145	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
8146	*
8147	* @returns VBox status code.
8148	* @param pVCpu The cross context virtual CPU structure.
8149	* @param fImportState Whether to import the guest state from the VMCS back
8150	* to the guest-CPU context.
8151	*
8152	* @remarks No-long-jmp zone!!!
8153	*/
8154	static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
8155	{
8156	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8157	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8158
8159	RTCPUID const idCpu = RTMpCpuId();
8160	Log4Func(("HostCpuId=%u\n", idCpu));
8161
8162	/*
8163	* !!! IMPORTANT !!!
8164	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
8165	*/
8166
8167	/* Save the guest state if necessary. */
8168	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8169	if (fImportState)
8170	{
8171	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8172	AssertRCReturn(rc, rc);
8173	}
8174
8175	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
8176	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8177	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8178
8179	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
8180	#ifdef VBOX_STRICT
8181	if (CPUMIsHyperDebugStateActive(pVCpu))
8182	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
8183	#endif
8184	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8185	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
8186	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
8187
8188	#if HC_ARCH_BITS == 64
8189	/* Restore host-state bits that VT-x only restores partially. */
8190	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8191	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8192	{
8193	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
8194	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8195	}
8196	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8197	#endif
8198
8199	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8200	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8201	{
8202	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
8203	if (!fImportState)
8204	{
8205	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
8206	AssertRCReturn(rc, rc);
8207	}
8208	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8209	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
8210	}
8211	else
8212	pVCpu->hm.s.vmx.fLazyMsrs = 0;
8213
8214	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8215	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8216
8217	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8218	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8219	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8220	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8221	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8222	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8223	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8224	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8225	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitVmentry);
8226	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8227
8228	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8229
8230	/** @todo This partially defeats the purpose of having preemption hooks.
8231	* The problem is, deregistering the hooks should be moved to a place that
8232	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8233	* context.
8234	*/
8235	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8236	AssertRCReturn(rc, rc);
8237
8238	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8239	/*
8240	* A valid shadow VMCS is made active as part of VM-entry. It is necessary to
8241	* clear a shadow VMCS before allowing that VMCS to become active on another
8242	* logical processor. We may or may not be importing guest state which clears
8243	* it, so cover for it here.
8244	*
8245	* See Intel spec. 24.11.1 "Software Use of Virtual-Machine Control Structures".
8246	*/
8247	if ( pVmcsInfo->pvShadowVmcs
8248	&& pVmcsInfo->fShadowVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
8249	{
8250	rc = hmR0VmxClearShadowVmcs(pVmcsInfo);
8251	AssertRCReturn(rc, rc);
8252	}
8253
8254	/*
8255	* Flag that we need to re-import the host state if we switch to this VMCS before
8256	* executing guest or nested-guest code.
8257	*/
8258	pVmcsInfo->idHostCpu = NIL_RTCPUID;
8259	#endif
8260
8261	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8262	NOREF(idCpu);
8263	return VINF_SUCCESS;
8264	}
8265
8266
8267	/**
8268	* Leaves the VT-x session.
8269	*
8270	* @returns VBox status code.
8271	* @param pVCpu The cross context virtual CPU structure.
8272	*
8273	* @remarks No-long-jmp zone!!!
8274	*/
8275	static int hmR0VmxLeaveSession(PVMCPU pVCpu)
8276	{
8277	HM_DISABLE_PREEMPT(pVCpu);
8278	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8279	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8280	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8281
8282	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8283	and done this from the VMXR0ThreadCtxCallback(). */
8284	if (!pVCpu->hm.s.fLeaveDone)
8285	{
8286	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8287	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8288	pVCpu->hm.s.fLeaveDone = true;
8289	}
8290	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8291
8292	/*
8293	* !!! IMPORTANT !!!
8294	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
8295	*/
8296
8297	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8298	/** @todo Deregistering here means we need to VMCLEAR always
8299	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8300	* for calling VMMR0ThreadCtxHookDisable here! */
8301	VMMR0ThreadCtxHookDisable(pVCpu);
8302
8303	/* Leave HM context. This takes care of local init (term). */
8304	int rc = HMR0LeaveCpu(pVCpu);
8305
8306	HM_RESTORE_PREEMPT();
8307	return rc;
8308	}
8309
8310
8311	/**
8312	* Does the necessary state syncing before doing a longjmp to ring-3.
8313	*
8314	* @returns VBox status code.
8315	* @param pVCpu The cross context virtual CPU structure.
8316	*
8317	* @remarks No-long-jmp zone!!!
8318	*/
8319	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
8320	{
8321	return hmR0VmxLeaveSession(pVCpu);
8322	}
8323
8324
8325	/**
8326	* Take necessary actions before going back to ring-3.
8327	*
8328	* An action requires us to go back to ring-3. This function does the necessary
8329	* steps before we can safely return to ring-3. This is not the same as longjmps
8330	* to ring-3, this is voluntary and prepares the guest so it may continue
8331	* executing outside HM (recompiler/IEM).
8332	*
8333	* @returns VBox status code.
8334	* @param pVCpu The cross context virtual CPU structure.
8335	* @param rcExit The reason for exiting to ring-3. Can be
8336	* VINF_VMM_UNKNOWN_RING3_CALL.
8337	*/
8338	static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
8339	{
8340	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8341
8342	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8343	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8344	{
8345	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8346	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8347	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8348	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8349	}
8350
8351	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8352	VMMRZCallRing3Disable(pVCpu);
8353	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8354
8355	/*
8356	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8357	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8358	*
8359	* This is because execution may continue from ring-3 and we would need to inject
8360	* the event from there (hence place it back in TRPM).
8361	*/
8362	if (pVCpu->hm.s.Event.fPending)
8363	{
8364	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8365	Assert(!pVCpu->hm.s.Event.fPending);
8366
8367	/* Clear the events from the VMCS. */
8368	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8369	AssertRCReturn(rc, rc);
8370	}
8371	#ifdef VBOX_STRICT
8372	else
8373	{
8374	/*
8375	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8376	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8377	* occasionally, see @bugref{9180#c42}.
8378	*
8379	* However, if the VM-entry failed, any VM entry-interruption info. field would
8380	* be left unmodified as the event would not have been injected to the guest. In
8381	* such cases, don't assert, we're not going to continue guest execution anyway.
8382	*/
8383	uint32_t uExitReason;
8384	uint32_t uEntryIntInfo;
8385	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8386	rc \|= VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8387	AssertRC(rc);
8388	Assert(VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason) \|\| !VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
8389	}
8390	#endif
8391
8392	/*
8393	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8394	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8395	* (e.g. TPR below threshold).
8396	*/
8397	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8398	{
8399	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8400	rc \|= hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8401	AssertRCReturn(rc, rc);
8402	}
8403
8404	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8405	and if we're injecting an event we should have a TRPM trap pending. */
8406	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8407	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8408	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8409	#endif
8410
8411	/* Save guest state and restore host state bits. */
8412	int rc = hmR0VmxLeaveSession(pVCpu);
8413	AssertRCReturn(rc, rc);
8414	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8415
8416	/* Thread-context hooks are unregistered at this point!!! */
8417
8418	/* Sync recompiler state. */
8419	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
8420	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
8421	\| CPUM_CHANGED_LDTR
8422	\| CPUM_CHANGED_GDTR
8423	\| CPUM_CHANGED_IDTR
8424	\| CPUM_CHANGED_TR
8425	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
8426	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
8427	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
8428	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
8429
8430	Assert(!pVCpu->hm.s.fClearTrapFlag);
8431
8432	/* Update the exit-to-ring 3 reason. */
8433	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
8434
8435	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
8436	if ( rcExit != VINF_EM_RAW_INTERRUPT
8437	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8438	{
8439	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
8440	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8441	}
8442
8443	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
8444
8445	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
8446	VMMRZCallRing3RemoveNotification(pVCpu);
8447	VMMRZCallRing3Enable(pVCpu);
8448
8449	return rc;
8450	}
8451
8452
8453	/**
8454	* VMMRZCallRing3() callback wrapper which saves the guest state before we
8455	* longjump to ring-3 and possibly get preempted.
8456	*
8457	* @returns VBox status code.
8458	* @param pVCpu The cross context virtual CPU structure.
8459	* @param enmOperation The operation causing the ring-3 longjump.
8460	* @param pvUser User argument, currently unused, NULL.
8461	*/
8462	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
8463	{
8464	RT_NOREF(pvUser);
8465	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
8466	{
8467	/*
8468	* !!! IMPORTANT !!!
8469	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
8470	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
8471	*/
8472	VMMRZCallRing3RemoveNotification(pVCpu);
8473	VMMRZCallRing3Disable(pVCpu);
8474	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
8475	RTThreadPreemptDisable(&PreemptState);
8476
8477	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8478	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8479	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8480	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8481
8482	#if HC_ARCH_BITS == 64
8483	/* Restore host-state bits that VT-x only restores partially. */
8484	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8485	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8486	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8487	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8488	#endif
8489
8490	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8491	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8492	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8493
8494	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8495	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8496	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8497
8498	/* Clear the current VMCS data back to memory (shadow VMCS if any would have been
8499	cleared as part of importing the guest state above. */
8500	hmR0VmxClearVmcs(pVmcsInfo);
8501
8502	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
8503	VMMR0ThreadCtxHookDisable(pVCpu);
8504	HMR0LeaveCpu(pVCpu);
8505	RTThreadPreemptRestore(&PreemptState);
8506	return VINF_SUCCESS;
8507	}
8508
8509	Assert(pVCpu);
8510	Assert(pvUser);
8511	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8512	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8513
8514	VMMRZCallRing3Disable(pVCpu);
8515	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8516
8517	Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
8518
8519	int rc = hmR0VmxLongJmpToRing3(pVCpu);
8520	AssertRCReturn(rc, rc);
8521
8522	VMMRZCallRing3Enable(pVCpu);
8523	return VINF_SUCCESS;
8524	}
8525
8526
8527	/**
8528	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8529	* stack.
8530	*
8531	* @returns Strict VBox status code (i.e. informational status codes too).
8532	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8533	* @param pVCpu The cross context virtual CPU structure.
8534	* @param uValue The value to push to the guest stack.
8535	*/
8536	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
8537	{
8538	/*
8539	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8540	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8541	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8542	*/
8543	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8544	if (pCtx->sp == 1)
8545	return VINF_EM_RESET;
8546	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8547	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
8548	AssertRC(rc);
8549	return rc;
8550	}
8551
8552
8553	/**
8554	* Injects an event into the guest upon VM-entry by updating the relevant fields
8555	* in the VM-entry area in the VMCS.
8556	*
8557	* @returns Strict VBox status code (i.e. informational status codes too).
8558	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8559	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8560	*
8561	* @param pVCpu The cross context virtual CPU structure.
8562	* @param pVmxTransient The VMX-transient structure.
8563	* @param pEvent The event being injected.
8564	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state. This
8565	* will be updated if necessary. This cannot not be NULL.
8566	* @param fStepping Whether we're single-stepping guest execution and should
8567	* return VINF_EM_DBG_STEPPED if the event is injected
8568	* directly (registers modified by us, not by hardware on
8569	* VM-entry).
8570	*/
8571	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
8572	uint32_t *pfIntrState)
8573	{
8574	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8575	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
8576	Assert(pfIntrState);
8577
8578	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8579	uint32_t u32IntInfo = pEvent->u64IntInfo;
8580	uint32_t const u32ErrCode = pEvent->u32ErrCode;
8581	uint32_t const cbInstr = pEvent->cbInstr;
8582	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
8583	uint8_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
8584	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
8585
8586	#ifdef VBOX_STRICT
8587	/*
8588	* Validate the error-code-valid bit for hardware exceptions.
8589	* No error codes for exceptions in real-mode.
8590	*
8591	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8592	*/
8593	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8594	&& !CPUMIsGuestInRealModeEx(pCtx))
8595	{
8596	switch (uVector)
8597	{
8598	case X86_XCPT_PF:
8599	case X86_XCPT_DF:
8600	case X86_XCPT_TS:
8601	case X86_XCPT_NP:
8602	case X86_XCPT_SS:
8603	case X86_XCPT_GP:
8604	case X86_XCPT_AC:
8605	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
8606	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8607	RT_FALL_THRU();
8608	default:
8609	break;
8610	}
8611	}
8612
8613	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8614	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
8615	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
8616	#endif
8617
8618	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8619
8620	/*
8621	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
8622	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
8623	* interrupt handler in the (real-mode) guest.
8624	*
8625	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
8626	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8627	*/
8628	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
8629	{
8630	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
8631	{
8632	/*
8633	* For CPUs with unrestricted guest execution enabled and with the guest
8634	* in real-mode, we must not set the deliver-error-code bit.
8635	*
8636	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8637	*/
8638	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
8639	}
8640	else
8641	{
8642	PVM pVM = pVCpu->CTX_SUFF(pVM);
8643	Assert(PDMVmmDevHeapIsEnabled(pVM));
8644	Assert(pVM->hm.s.vmx.pRealModeTSS);
8645	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
8646
8647	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
8648	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8649	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
8650	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
8651	AssertRCReturn(rc2, rc2);
8652
8653	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8654	size_t const cbIdtEntry = sizeof(X86IDTR16);
8655	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
8656	{
8657	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8658	if (uVector == X86_XCPT_DF)
8659	return VINF_EM_RESET;
8660
8661	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
8662	No error codes for exceptions in real-mode. */
8663	if (uVector == X86_XCPT_GP)
8664	{
8665	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
8666	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8667	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8668	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8669	HMEVENT EventXcptDf;
8670	RT_ZERO(EventXcptDf);
8671	EventXcptDf.u64IntInfo = uXcptDfInfo;
8672	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
8673	}
8674
8675	/*
8676	* If we're injecting an event with no valid IDT entry, inject a #GP.
8677	* No error codes for exceptions in real-mode.
8678	*
8679	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8680	*/
8681	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
8682	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8683	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8684	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8685	HMEVENT EventXcptGp;
8686	RT_ZERO(EventXcptGp);
8687	EventXcptGp.u64IntInfo = uXcptGpInfo;
8688	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
8689	}
8690
8691	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8692	uint16_t uGuestIp = pCtx->ip;
8693	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
8694	{
8695	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8696	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8697	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8698	}
8699	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
8700	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8701
8702	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8703	X86IDTR16 IdtEntry;
8704	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
8705	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8706	AssertRCReturn(rc2, rc2);
8707
8708	/* Construct the stack frame for the interrupt/exception handler. */
8709	VBOXSTRICTRC rcStrict;
8710	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
8711	if (rcStrict == VINF_SUCCESS)
8712	{
8713	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
8714	if (rcStrict == VINF_SUCCESS)
8715	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
8716	}
8717
8718	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8719	if (rcStrict == VINF_SUCCESS)
8720	{
8721	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8722	pCtx->rip = IdtEntry.offSel;
8723	pCtx->cs.Sel = IdtEntry.uSel;
8724	pCtx->cs.ValidSel = IdtEntry.uSel;
8725	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8726	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
8727	&& uVector == X86_XCPT_PF)
8728	pCtx->cr2 = GCPtrFault;
8729
8730	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
8731	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
8732	\| HM_CHANGED_GUEST_RSP);
8733
8734	/*
8735	* If we delivered a hardware exception (other than an NMI) and if there was
8736	* block-by-STI in effect, we should clear it.
8737	*/
8738	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
8739	{
8740	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
8741	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
8742	Log4Func(("Clearing inhibition due to STI\n"));
8743	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
8744	}
8745
8746	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8747	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
8748
8749	/*
8750	* The event has been truly dispatched to the guest. Mark it as no longer pending so
8751	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
8752	*/
8753	pVCpu->hm.s.Event.fPending = false;
8754
8755	/*
8756	* If we eventually support nested-guest execution without unrestricted guest execution,
8757	* we should set fInterceptEvents here.
8758	*/
8759	Assert(!pVmxTransient->fIsNestedGuest);
8760
8761	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
8762	if (fStepping)
8763	rcStrict = VINF_EM_DBG_STEPPED;
8764	}
8765	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8766	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8767	return rcStrict;
8768	}
8769	}
8770
8771	/*
8772	* Validate.
8773	*/
8774	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8775	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
8776
8777	/*
8778	* Inject the event into the VMCS.
8779	*/
8780	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8781	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8782	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8783	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8784	AssertRCReturn(rc, rc);
8785
8786	/*
8787	* Update guest CR2 if this is a page-fault.
8788	*/
8789	if (VMX_ENTRY_INT_INFO_IS_XCPT_PF(u32IntInfo))
8790	pCtx->cr2 = GCPtrFault;
8791
8792	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8793	return VINF_SUCCESS;
8794	}
8795
8796
8797	/**
8798	* Evaluates the event to be delivered to the guest and sets it as the pending
8799	* event.
8800	*
8801	* @returns Strict VBox status code (i.e. informational status codes too).
8802	* @param pVCpu The cross context virtual CPU structure.
8803	* @param pVmxTransient The VMX-transient structure.
8804	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
8805	*/
8806	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
8807	{
8808	Assert(pfIntrState);
8809	Assert(!TRPMHasTrap(pVCpu));
8810
8811	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8812	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8813	bool const fIsNestedGuest = pVmxTransient->fIsNestedGuest;
8814
8815	/*
8816	* Get the current interruptibility-state of the guest or nested-guest and
8817	* then figure out what needs to be injected.
8818	*/
8819	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmxTransient);
8820	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8821	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8822	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8823
8824	/* We don't support block-by-SMI yet.*/
8825	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI));
8826
8827	/* Block-by-STI must not be set when interrupts are disabled. */
8828	if (fBlockSti)
8829	{
8830	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
8831	Assert(pCtx->eflags.Bits.u1IF);
8832	}
8833
8834	/* Update interruptibility state to the caller. */
8835	*pfIntrState = fIntrState;
8836
8837	/*
8838	* Toggling of interrupt force-flags here is safe since we update TRPM on
8839	* premature exits to ring-3 before executing guest code, see hmR0VmxExitToRing3().
8840	* We must NOT restore these force-flags.
8841	*/
8842
8843	/** @todo SMI. SMIs take priority over NMIs. */
8844
8845	/*
8846	* Check if an NMI is pending and if the guest or nested-guest can receive them.
8847	* NMIs take priority over external interrupts.
8848	*/
8849	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
8850	{
8851	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
8852	if ( !pVCpu->hm.s.Event.fPending
8853	&& !fBlockNmi
8854	&& !fBlockSti
8855	&& !fBlockMovSS)
8856	{
8857	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8858	if ( fIsNestedGuest
8859	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
8860	return IEMExecVmxVmexitXcptNmi(pVCpu);
8861	#endif
8862	hmR0VmxSetPendingXcptNmi(pVCpu);
8863	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
8864	Log4Func(("Pending NMI\n"));
8865	}
8866	else if (!fIsNestedGuest)
8867	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
8868	/* else: for nested-guests, NMI-window exiting will be picked up when merging VMCS controls. */
8869	}
8870	/*
8871	* Check if an external interrupt (PIC/APIC) is pending and if the guest or nested-guest
8872	* can receive them. Once PDMGetInterrupt() returns a valid interrupt we -must- deliver
8873	* the interrupt. We can no longer re-request it from the APIC.
8874	*/
8875	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
8876	&& !pVCpu->hm.s.fSingleInstruction)
8877	{
8878	Assert(!DBGFIsStepping(pVCpu));
8879	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
8880	AssertRCReturn(rc, rc);
8881
8882	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
8883	if ( !pVCpu->hm.s.Event.fPending
8884	&& !fBlockInt
8885	&& !fBlockSti
8886	&& !fBlockMovSS)
8887	{
8888	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8889	if ( fIsNestedGuest
8890	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8891	&& !CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8892	{
8893	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0 /* uVector /, true / fIntPending */);
8894	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
8895	return rcStrict;
8896	}
8897	#endif
8898	uint8_t u8Interrupt;
8899	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
8900	if (RT_SUCCESS(rc))
8901	{
8902	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8903	if ( fIsNestedGuest
8904	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8905	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8906	{
8907	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
8908	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
8909	return rcStrict;
8910	}
8911	#endif
8912	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
8913	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
8914	}
8915	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
8916	{
8917	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
8918
8919	if ( !fIsNestedGuest
8920	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
8921	hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
8922	/* else: for nested-guests, TPR threshold is picked up while merging VMCS controls. */
8923
8924	/*
8925	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
8926	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
8927	* need to re-set this force-flag here.
8928	*/
8929	}
8930	else
8931	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
8932	}
8933	else if (!fIsNestedGuest)
8934	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
8935	/* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCS controls. */
8936	}
8937
8938	return VINF_SUCCESS;
8939	}
8940
8941
8942	/**
8943	* Injects any pending events into the guest if the guest is in a state to
8944	* receive them.
8945	*
8946	* @returns Strict VBox status code (i.e. informational status codes too).
8947	* @param pVCpu The cross context virtual CPU structure.
8948	* @param pVmxTransient The VMX-transient structure.
8949	* @param fIntrState The VT-x guest-interruptibility state.
8950	* @param fStepping Whether we are single-stepping the guest using the
8951	* hypervisor debugger and should return
8952	* VINF_EM_DBG_STEPPED if the event was dispatched
8953	* directly.
8954	*/
8955	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
8956	{
8957	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8958	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8959
8960	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8961	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8962
8963	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8964	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8965	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8966	Assert(!TRPMHasTrap(pVCpu));
8967
8968	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8969	if (pVCpu->hm.s.Event.fPending)
8970	{
8971	/*
8972	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
8973	* pending even while injecting an event and in this case, we want a VM-exit as soon as
8974	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
8975	*
8976	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
8977	*/
8978	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8979	#ifdef VBOX_STRICT
8980	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8981	{
8982	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
8983	Assert(!fBlockInt);
8984	Assert(!fBlockSti);
8985	Assert(!fBlockMovSS);
8986	}
8987	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
8988	{
8989	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8990	Assert(!fBlockSti);
8991	Assert(!fBlockMovSS);
8992	Assert(!fBlockNmi);
8993	}
8994	#endif
8995	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
8996	uIntType));
8997
8998	/*
8999	* Inject the event and get any changes to the guest-interruptibility state.
9000	*
9001	* The guest-interruptibility state may need to be updated if we inject the event
9002	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
9003	*/
9004	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
9005	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
9006
9007	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
9008	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
9009	else
9010	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
9011	}
9012
9013	/*
9014	* Update the guest-interruptibility state.
9015	*
9016	* This is required for the real-on-v86 software interrupt injection case above, as well as
9017	* updates to the guest state from ring-3 or IEM/REM.
9018	*/
9019	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
9020	AssertRCReturn(rc, rc);
9021
9022	/*
9023	* There's no need to clear the VM-entry interruption-information field here if we're not
9024	* injecting anything. VT-x clears the valid bit on every VM-exit.
9025	*
9026	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
9027	*/
9028
9029	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
9030	NOREF(fBlockMovSS); NOREF(fBlockSti);
9031	return rcStrict;
9032	}
9033
9034
9035	/**
9036	* Enters the VT-x session.
9037	*
9038	* @returns VBox status code.
9039	* @param pVCpu The cross context virtual CPU structure.
9040	*/
9041	VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu)
9042	{
9043	AssertPtr(pVCpu);
9044	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
9045	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9046
9047	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9048	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9049	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9050
9051	#ifdef VBOX_STRICT
9052	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
9053	RTCCUINTREG uHostCr4 = ASMGetCR4();
9054	if (!(uHostCr4 & X86_CR4_VMXE))
9055	{
9056	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
9057	return VERR_VMX_X86_CR4_VMXE_CLEARED;
9058	}
9059	#endif
9060
9061	/*
9062	* Load the appropriate VMCS as the current and active one.
9063	*/
9064	PVMXVMCSINFO pVmcsInfo;
9065	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
9066	if (!fInNestedGuestMode)
9067	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
9068	else
9069	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
9070	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
9071	if (RT_SUCCESS(rc))
9072	{
9073	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
9074	pVCpu->hm.s.fLeaveDone = false;
9075	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9076
9077	/*
9078	* Do the EMT scheduled L1D flush here if needed.
9079	*/
9080	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9081	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9082	else if (pVCpu->CTX_SUFF(pVM)->hm.s.fMdsClearOnSched)
9083	hmR0MdsClear();
9084	}
9085	return rc;
9086	}
9087
9088
9089	/**
9090	* The thread-context callback (only on platforms which support it).
9091	*
9092	* @param enmEvent The thread-context event.
9093	* @param pVCpu The cross context virtual CPU structure.
9094	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
9095	* @thread EMT(pVCpu)
9096	*/
9097	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
9098	{
9099	AssertPtr(pVCpu);
9100	RT_NOREF1(fGlobalInit);
9101
9102	switch (enmEvent)
9103	{
9104	case RTTHREADCTXEVENT_OUT:
9105	{
9106	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9107	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9108	VMCPU_ASSERT_EMT(pVCpu);
9109
9110	/* No longjmps (logger flushes, locks) in this fragile context. */
9111	VMMRZCallRing3Disable(pVCpu);
9112	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
9113
9114	/* Restore host-state (FPU, debug etc.) */
9115	if (!pVCpu->hm.s.fLeaveDone)
9116	{
9117	/*
9118	* Do -not- import the guest-state here as we might already be in the middle of importing
9119	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
9120	*/
9121	hmR0VmxLeave(pVCpu, false /* fImportState */);
9122	pVCpu->hm.s.fLeaveDone = true;
9123	}
9124
9125	/* Leave HM context, takes care of local init (term). */
9126	int rc = HMR0LeaveCpu(pVCpu);
9127	AssertRC(rc);
9128
9129	/* Restore longjmp state. */
9130	VMMRZCallRing3Enable(pVCpu);
9131	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
9132	break;
9133	}
9134
9135	case RTTHREADCTXEVENT_IN:
9136	{
9137	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9138	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
9139	VMCPU_ASSERT_EMT(pVCpu);
9140
9141	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
9142	VMMRZCallRing3Disable(pVCpu);
9143	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
9144
9145	/* Initialize the bare minimum state required for HM. This takes care of
9146	initializing VT-x if necessary (onlined CPUs, local init etc.) */
9147	int rc = hmR0EnterCpu(pVCpu);
9148	AssertRC(rc);
9149	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9150	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
9151
9152	/* Load the active VMCS as the current one. */
9153	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
9154	rc = hmR0VmxLoadVmcs(pVmcsInfo);
9155	AssertRC(rc);
9156	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
9157	pVCpu->hm.s.fLeaveDone = false;
9158
9159	/* Do the EMT scheduled L1D flush if needed. */
9160	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
9161	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
9162
9163	/* Restore longjmp state. */
9164	VMMRZCallRing3Enable(pVCpu);
9165	break;
9166	}
9167
9168	default:
9169	break;
9170	}
9171	}
9172
9173
9174	/**
9175	* Exports the host state into the VMCS host-state area.
9176	* Sets up the VM-exit MSR-load area.
9177	*
9178	* The CPU state will be loaded from these fields on every successful VM-exit.
9179	*
9180	* @returns VBox status code.
9181	* @param pVCpu The cross context virtual CPU structure.
9182	*
9183	* @remarks No-long-jump zone!!!
9184	*/
9185	static int hmR0VmxExportHostState(PVMCPU pVCpu)
9186	{
9187	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9188
9189	int rc = VINF_SUCCESS;
9190	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
9191	{
9192	rc = hmR0VmxExportHostControlRegs();
9193	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9194
9195	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
9196	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9197
9198	rc = hmR0VmxExportHostMsrs(pVCpu);
9199	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9200
9201	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
9202	}
9203	return rc;
9204	}
9205
9206
9207	/**
9208	* Saves the host state in the VMCS host-state.
9209	*
9210	* @returns VBox status code.
9211	* @param pVCpu The cross context virtual CPU structure.
9212	*
9213	* @remarks No-long-jump zone!!!
9214	*/
9215	VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
9216	{
9217	AssertPtr(pVCpu);
9218	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9219
9220	/*
9221	* Export the host state here while entering HM context.
9222	* When thread-context hooks are used, we might get preempted and have to re-save the host
9223	* state but most of the time we won't be, so do it here before we disable interrupts.
9224	*/
9225	return hmR0VmxExportHostState(pVCpu);
9226	}
9227
9228
9229	/**
9230	* Exports the guest state into the VMCS guest-state area.
9231	*
9232	* The will typically be done before VM-entry when the guest-CPU state and the
9233	* VMCS state may potentially be out of sync.
9234	*
9235	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9236	* VM-entry controls.
9237	* Sets up the appropriate VMX non-root function to execute guest code based on
9238	* the guest CPU mode.
9239	*
9240	* @returns VBox strict status code.
9241	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9242	* without unrestricted guest execution and the VMMDev is not presently
9243	* mapped (e.g. EFI32).
9244	*
9245	* @param pVCpu The cross context virtual CPU structure.
9246	* @param pVmxTransient The VMX-transient structure.
9247	*
9248	* @remarks No-long-jump zone!!!
9249	*/
9250	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9251	{
9252	AssertPtr(pVCpu);
9253	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9254	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9255
9256	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9257
9258	/*
9259	* Determine real-on-v86 mode.
9260	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9261	*/
9262	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9263	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9264	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9265	pVmcsInfo->RealMode. fRealOnV86Active = false;
9266	else
9267	{
9268	Assert(!pVmxTransient->fIsNestedGuest);
9269	pVmcsInfo->RealMode.fRealOnV86Active = true;
9270	}
9271
9272	/*
9273	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9274	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9275	*/
9276	/** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
9277	* hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
9278	* be a need to evaluate this everytime since I'm pretty sure we intercept
9279	* all guest paging mode changes. */
9280	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
9281	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9282
9283	rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9284	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9285
9286	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9287	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9288
9289	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9290	if (rcStrict == VINF_SUCCESS)
9291	{ /* likely */ }
9292	else
9293	{
9294	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9295	return rcStrict;
9296	}
9297
9298	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9299	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9300
9301	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9302	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9303
9304	rc = hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9305	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9306
9307	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9308	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9309
9310	rc = hmR0VmxExportGuestRip(pVCpu);
9311	rc \|= hmR0VmxExportGuestRsp(pVCpu);
9312	rc \|= hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9313	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9314
9315	rc = hmR0VmxExportGuestHwvirtState(pVCpu, pVmxTransient);
9316	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9317
9318	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9319	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9320	\| HM_CHANGED_GUEST_CR2
9321	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9322	\| HM_CHANGED_GUEST_X87
9323	\| HM_CHANGED_GUEST_SSE_AVX
9324	\| HM_CHANGED_GUEST_OTHER_XSAVE
9325	\| HM_CHANGED_GUEST_XCRx
9326	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9327	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9328	\| HM_CHANGED_GUEST_TSC_AUX
9329	\| HM_CHANGED_GUEST_OTHER_MSRS
9330	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9331
9332	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9333	return rc;
9334	}
9335
9336
9337	/**
9338	* Exports the state shared between the host and guest into the VMCS.
9339	*
9340	* @param pVCpu The cross context virtual CPU structure.
9341	* @param pVmxTransient The VMX-transient structure.
9342	*
9343	* @remarks No-long-jump zone!!!
9344	*/
9345	static void hmR0VmxExportSharedState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9346	{
9347	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9348	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9349
9350	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9351	{
9352	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9353	AssertRC(rc);
9354	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9355
9356	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9357	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9358	{
9359	rc = hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9360	AssertRC(rc);
9361	}
9362	}
9363
9364	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9365	{
9366	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9367	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9368	}
9369
9370	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9371	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9372	}
9373
9374
9375	/**
9376	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9377	*
9378	* @returns Strict VBox status code (i.e. informational status codes too).
9379	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9380	* without unrestricted guest execution and the VMMDev is not presently
9381	* mapped (e.g. EFI32).
9382	*
9383	* @param pVCpu The cross context virtual CPU structure.
9384	* @param pVmxTransient The VMX-transient structure.
9385	*
9386	* @remarks No-long-jump zone!!!
9387	*/
9388	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9389	{
9390	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9391	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9392	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9393
9394	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9395	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9396	#endif
9397
9398	/*
9399	* For many exits it's only RIP that changes and hence try to export it first
9400	* without going through a lot of change flag checks.
9401	*/
9402	VBOXSTRICTRC rcStrict;
9403	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9404	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9405	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
9406	{
9407	rcStrict = hmR0VmxExportGuestRip(pVCpu);
9408	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9409	{ /* likely */}
9410	else
9411	AssertMsgFailedReturn(("Failed to export guest RIP! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
9412	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9413	}
9414	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9415	{
9416	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9417	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9418	{ /* likely */}
9419	else
9420	{
9421	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9422	VBOXSTRICTRC_VAL(rcStrict)));
9423	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9424	return rcStrict;
9425	}
9426	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9427	}
9428	else
9429	rcStrict = VINF_SUCCESS;
9430
9431	#ifdef VBOX_STRICT
9432	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9433	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9434	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9435	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
9436	("fCtxChanged=%#RX64\n", fCtxChanged));
9437	#endif
9438	return rcStrict;
9439	}
9440
9441
9442	/**
9443	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9444	* and update error record fields accordingly.
9445	*
9446	* @returns VMX_IGS_* error codes.
9447	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9448	* wrong with the guest state.
9449	*
9450	* @param pVCpu The cross context virtual CPU structure.
9451	* @param pVmcsInfo The VMCS info. object.
9452	*
9453	* @remarks This function assumes our cache of the VMCS controls
9454	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9455	*/
9456	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
9457	{
9458	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9459	#define HMVMX_CHECK_BREAK(expr, err) do { \
9460	if (!(expr)) { uError = (err); break; } \
9461	} while (0)
9462
9463	int rc;
9464	PVM pVM = pVCpu->CTX_SUFF(pVM);
9465	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9466	uint32_t uError = VMX_IGS_ERROR;
9467	uint32_t u32Val;
9468	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9469
9470	do
9471	{
9472	/*
9473	* CR0.
9474	*/
9475	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
9476	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9477	uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9478	/* Exceptions for unrestricted guest execution for fixed CR0 bits (PE, PG).
9479	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9480	if (fUnrestrictedGuest)
9481	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
9482
9483	uint32_t u32GuestCr0;
9484	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
9485	AssertRCBreak(rc);
9486	HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
9487	HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
9488	if ( !fUnrestrictedGuest
9489	&& (u32GuestCr0 & X86_CR0_PG)
9490	&& !(u32GuestCr0 & X86_CR0_PE))
9491	{
9492	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9493	}
9494
9495	/*
9496	* CR4.
9497	*/
9498	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
9499	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9500	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9501
9502	uint32_t u32GuestCr4;
9503	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
9504	AssertRCBreak(rc);
9505	HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
9506	HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
9507
9508	/*
9509	* IA32_DEBUGCTL MSR.
9510	*/
9511	uint64_t u64Val;
9512	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9513	AssertRCBreak(rc);
9514	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9515	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9516	{
9517	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9518	}
9519	uint64_t u64DebugCtlMsr = u64Val;
9520
9521	#ifdef VBOX_STRICT
9522	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9523	AssertRCBreak(rc);
9524	Assert(u32Val == pVmcsInfo->u32EntryCtls);
9525	#endif
9526	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
9527
9528	/*
9529	* RIP and RFLAGS.
9530	*/
9531	uint32_t u32Eflags;
9532	#if HC_ARCH_BITS == 64
9533	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9534	AssertRCBreak(rc);
9535	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9536	if ( !fLongModeGuest
9537	\|\| !pCtx->cs.Attr.n.u1Long)
9538	{
9539	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9540	}
9541	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9542	* must be identical if the "IA-32e mode guest" VM-entry
9543	* control is 1 and CS.L is 1. No check applies if the
9544	* CPU supports 64 linear-address bits. */
9545
9546	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9547	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9548	AssertRCBreak(rc);
9549	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9550	VMX_IGS_RFLAGS_RESERVED);
9551	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9552	u32Eflags = u64Val;
9553	#else
9554	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9555	AssertRCBreak(rc);
9556	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9557	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9558	#endif
9559
9560	if ( fLongModeGuest
9561	\|\| ( fUnrestrictedGuest
9562	&& !(u32GuestCr0 & X86_CR0_PE)))
9563	{
9564	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9565	}
9566
9567	uint32_t u32EntryInfo;
9568	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9569	AssertRCBreak(rc);
9570	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
9571	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9572
9573	/*
9574	* 64-bit checks.
9575	*/
9576	#if HC_ARCH_BITS == 64
9577	if (fLongModeGuest)
9578	{
9579	HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9580	HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9581	}
9582
9583	if ( !fLongModeGuest
9584	&& (u32GuestCr4 & X86_CR4_PCIDE))
9585	{
9586	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9587	}
9588
9589	/** @todo CR3 field must be such that bits 63:52 and bits in the range
9590	* 51:32 beyond the processor's physical-address width are 0. */
9591
9592	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9593	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9594	{
9595	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9596	}
9597
9598	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9599	AssertRCBreak(rc);
9600	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9601
9602	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9603	AssertRCBreak(rc);
9604	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9605	#endif
9606
9607	/*
9608	* PERF_GLOBAL MSR.
9609	*/
9610	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
9611	{
9612	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9613	AssertRCBreak(rc);
9614	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9615	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9616	}
9617
9618	/*
9619	* PAT MSR.
9620	*/
9621	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
9622	{
9623	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9624	AssertRCBreak(rc);
9625	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9626	for (unsigned i = 0; i < 8; i++)
9627	{
9628	uint8_t u8Val = (u64Val & 0xff);
9629	if ( u8Val != 0 /* UC */
9630	&& u8Val != 1 /* WC */
9631	&& u8Val != 4 /* WT */
9632	&& u8Val != 5 /* WP */
9633	&& u8Val != 6 /* WB */
9634	&& u8Val != 7 /* UC- */)
9635	{
9636	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9637	}
9638	u64Val >>= 8;
9639	}
9640	}
9641
9642	/*
9643	* EFER MSR.
9644	*/
9645	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
9646	{
9647	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9648	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9649	AssertRCBreak(rc);
9650	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9651	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9652	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
9653	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
9654	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9655	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
9656	* iemVmxVmentryCheckGuestState(). */
9657	HMVMX_CHECK_BREAK( fUnrestrictedGuest
9658	\|\| !(u32GuestCr0 & X86_CR0_PG)
9659	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9660	VMX_IGS_EFER_LMA_LME_MISMATCH);
9661	}
9662
9663	/*
9664	* Segment registers.
9665	*/
9666	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9667	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9668	if (!(u32Eflags & X86_EFL_VM))
9669	{
9670	/* CS */
9671	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9672	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9673	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9674	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9675	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9676	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9677	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9678	/* CS cannot be loaded with NULL in protected mode. */
9679	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9680	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9681	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
9682	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9683	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
9684	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9685	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9686	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9687	else
9688	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9689
9690	/* SS */
9691	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9692	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9693	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9694	if ( !(pCtx->cr0 & X86_CR0_PE)
9695	\|\| pCtx->cs.Attr.n.u4Type == 3)
9696	{
9697	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9698	}
9699	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9700	{
9701	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9702	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9703	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9704	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9705	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9706	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9707	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9708	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9709	}
9710
9711	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
9712	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9713	{
9714	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9715	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9716	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9717	\|\| pCtx->ds.Attr.n.u4Type > 11
9718	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9719	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9720	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9721	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9722	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9723	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9724	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9725	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9726	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9727	}
9728	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9729	{
9730	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9731	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9732	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9733	\|\| pCtx->es.Attr.n.u4Type > 11
9734	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9735	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9736	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9737	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9738	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9739	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9740	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9741	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9742	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9743	}
9744	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9745	{
9746	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9747	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9748	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9749	\|\| pCtx->fs.Attr.n.u4Type > 11
9750	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9751	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9752	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9753	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9754	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9755	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9756	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9757	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9758	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9759	}
9760	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9761	{
9762	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9763	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9764	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9765	\|\| pCtx->gs.Attr.n.u4Type > 11
9766	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9767	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9768	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9769	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9770	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9771	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9772	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9773	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9774	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9775	}
9776	/* 64-bit capable CPUs. */
9777	#if HC_ARCH_BITS == 64
9778	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9779	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9780	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9781	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9782	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9783	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9784	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9785	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9786	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9787	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9788	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9789	#endif
9790	}
9791	else
9792	{
9793	/* V86 mode checks. */
9794	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9795	if (pVmcsInfo->RealMode.fRealOnV86Active)
9796	{
9797	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9798	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9799	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9800	}
9801	else
9802	{
9803	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9804	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9805	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9806	}
9807
9808	/* CS */
9809	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9810	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9811	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9812	/* SS */
9813	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9814	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9815	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9816	/* DS */
9817	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9818	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9819	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9820	/* ES */
9821	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9822	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9823	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9824	/* FS */
9825	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9826	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9827	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9828	/* GS */
9829	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9830	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9831	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9832	/* 64-bit capable CPUs. */
9833	#if HC_ARCH_BITS == 64
9834	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9835	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9836	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9837	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9838	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9839	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9840	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9841	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9842	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9843	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9844	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9845	#endif
9846	}
9847
9848	/*
9849	* TR.
9850	*/
9851	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9852	/* 64-bit capable CPUs. */
9853	#if HC_ARCH_BITS == 64
9854	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9855	#endif
9856	if (fLongModeGuest)
9857	{
9858	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9859	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9860	}
9861	else
9862	{
9863	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9864	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9865	VMX_IGS_TR_ATTR_TYPE_INVALID);
9866	}
9867	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9868	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9869	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9870	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9871	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9872	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9873	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9874	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9875
9876	/*
9877	* GDTR and IDTR.
9878	*/
9879	#if HC_ARCH_BITS == 64
9880	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9881	AssertRCBreak(rc);
9882	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9883
9884	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9885	AssertRCBreak(rc);
9886	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9887	#endif
9888
9889	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9890	AssertRCBreak(rc);
9891	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9892
9893	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9894	AssertRCBreak(rc);
9895	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9896
9897	/*
9898	* Guest Non-Register State.
9899	*/
9900	/* Activity State. */
9901	uint32_t u32ActivityState;
9902	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9903	AssertRCBreak(rc);
9904	HMVMX_CHECK_BREAK( !u32ActivityState
9905	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
9906	VMX_IGS_ACTIVITY_STATE_INVALID);
9907	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9908	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9909	uint32_t u32IntrState;
9910	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
9911	AssertRCBreak(rc);
9912	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
9913	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9914	{
9915	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9916	}
9917
9918	/** @todo Activity state and injecting interrupts. Left as a todo since we
9919	* currently don't use activity states but ACTIVE. */
9920
9921	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9922	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9923
9924	/* Guest interruptibility-state. */
9925	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9926	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
9927	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9928	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9929	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9930	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9931	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9932	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
9933	{
9934	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9935	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9936	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9937	}
9938	else if (VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
9939	{
9940	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9941	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9942	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9943	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9944	}
9945	/** @todo Assumes the processor is not in SMM. */
9946	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9947	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9948	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9949	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9950	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9951	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
9952	&& VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
9953	{
9954	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
9955	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9956	}
9957
9958	/* Pending debug exceptions. */
9959	#if HC_ARCH_BITS == 64
9960	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
9961	AssertRCBreak(rc);
9962	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9963	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9964	u32Val = u64Val; /* For pending debug exceptions checks below. */
9965	#else
9966	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
9967	AssertRCBreak(rc);
9968	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9969	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9970	#endif
9971
9972	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9973	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
9974	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9975	{
9976	if ( (u32Eflags & X86_EFL_TF)
9977	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9978	{
9979	/* Bit 14 is PendingDebug.BS. */
9980	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9981	}
9982	if ( !(u32Eflags & X86_EFL_TF)
9983	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9984	{
9985	/* Bit 14 is PendingDebug.BS. */
9986	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9987	}
9988	}
9989
9990	/* VMCS link pointer. */
9991	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9992	AssertRCBreak(rc);
9993	if (u64Val != UINT64_C(0xffffffffffffffff))
9994	{
9995	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9996	/** @todo Bits beyond the processor's physical-address width MBZ. */
9997	/** @todo SMM checks. */
9998	Assert(pVmcsInfo->HCPhysShadowVmcs == u64Val);
9999	Assert(pVmcsInfo->pvShadowVmcs);
10000	VMXVMCSREVID VmcsRevId;
10001	VmcsRevId.u = (uint32_t )pVmcsInfo->pvShadowVmcs;
10002	HMVMX_CHECK_BREAK(VmcsRevId.n.u31RevisionId == RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID),
10003	VMX_IGS_VMCS_LINK_PTR_SHADOW_VMCS_ID_INVALID);
10004	HMVMX_CHECK_BREAK(VmcsRevId.n.fIsShadowVmcs == (uint32_t)!!(pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING),
10005	VMX_IGS_VMCS_LINK_PTR_NOT_SHADOW);
10006	}
10007
10008	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10009	* not using nested paging? */
10010	if ( pVM->hm.s.fNestedPaging
10011	&& !fLongModeGuest
10012	&& CPUMIsGuestInPAEModeEx(pCtx))
10013	{
10014	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10015	AssertRCBreak(rc);
10016	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10017
10018	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10019	AssertRCBreak(rc);
10020	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10021
10022	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10023	AssertRCBreak(rc);
10024	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10025
10026	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10027	AssertRCBreak(rc);
10028	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10029	}
10030
10031	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10032	if (uError == VMX_IGS_ERROR)
10033	uError = VMX_IGS_REASON_NOT_FOUND;
10034	} while (0);
10035
10036	pVCpu->hm.s.u32HMError = uError;
10037	return uError;
10038
10039	#undef HMVMX_ERROR_BREAK
10040	#undef HMVMX_CHECK_BREAK
10041	}
10042
10043
10044	/**
10045	* Map the APIC-access page for virtualizing APIC accesses.
10046	*
10047	* This can cause a longjumps to R3 due to the acquisition of the PGM lock. Hence,
10048	* this not done as part of exporting guest state, see @bugref{8721}.
10049	*
10050	* @returns VBox status code.
10051	* @param pVCpu The cross context virtual CPU structure.
10052	*/
10053	static int hmR0VmxMapHCApicAccessPage(PVMCPU pVCpu)
10054	{
10055	PVM pVM = pVCpu->CTX_SUFF(pVM);
10056	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
10057
10058	Assert(PDMHasApic(pVM));
10059	Assert(u64MsrApicBase);
10060
10061	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
10062	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
10063
10064	/* Unalias the existing mapping. */
10065	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
10066	AssertRCReturn(rc, rc);
10067
10068	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
10069	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
10070	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
10071	AssertRCReturn(rc, rc);
10072
10073	/* Update the per-VCPU cache of the APIC base MSR. */
10074	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
10075	return VINF_SUCCESS;
10076	}
10077
10078
10079	/**
10080	* Worker function passed to RTMpOnSpecific() that is to be called on the target
10081	* CPU.
10082	*
10083	* @param idCpu The ID for the CPU the function is called on.
10084	* @param pvUser1 Null, not used.
10085	* @param pvUser2 Null, not used.
10086	*/
10087	static DECLCALLBACK(void) hmR0DispatchHostNmi(RTCPUID idCpu, void pvUser1, void pvUser2)
10088	{
10089	RT_NOREF3(idCpu, pvUser1, pvUser2);
10090	VMXDispatchHostNmi();
10091	}
10092
10093
10094	/**
10095	* Dispatching an NMI on the host CPU that received it.
10096	*
10097	* @returns VBox status code.
10098	* @param pVCpu The cross context virtual CPU structure.
10099	* @param pVmcsInfo The VMCS info. object corresponding to the VMCS that was
10100	* executing when receiving the host NMI in VMX non-root
10101	* operation.
10102	*/
10103	static int hmR0VmxExitHostNmi(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
10104	{
10105	RTCPUID const idCpu = pVmcsInfo->idHostCpu;
10106
10107	/*
10108	* We don't want to delay dispatching the NMI any more than we have to. However,
10109	* we have already chosen -not- to dispatch NMIs when interrupts were still disabled
10110	* after executing guest or nested-guest code for the following reasons:
10111	*
10112	* - We would need to perform VMREADs with interrupts disabled and is orders of
10113	* magnitude worse when we run as a guest hypervisor without VMCS shadowing
10114	* supported by the host hypervisor.
10115	*
10116	* - It affects the common VM-exit scenario and keeps interrupts disabled for a
10117	* longer period of time just for handling an edge case like host NMIs which do
10118	* not occur nearly as frequently as other VM-exits.
10119	*
10120	* Let's cover the most likely scenario first. Check if we are on the target CPU
10121	* and dispatch the NMI right away. This should be much faster than calling into
10122	* RTMpOnSpecific() machinery.
10123	*/
10124	bool fDispatched = false;
10125	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
10126	if (idCpu == RTMpCpuId())
10127	{
10128	VMXDispatchHostNmi();
10129	fDispatched = true;
10130	}
10131	ASMSetFlags(fEFlags);
10132	if (fDispatched)
10133	{
10134	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
10135	return VINF_SUCCESS;
10136	}
10137
10138	/*
10139	* RTMpOnSpecific() waits until the worker function has run on the target CPU. So
10140	* there should be no race or recursion even if we are unlucky enough to be preempted
10141	* (to the target CPU) without dispatching the host NMI above.
10142	*/
10143	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGCIpi);
10144	return RTMpOnSpecific(idCpu, &hmR0DispatchHostNmi, NULL /* pvUser1 /, NULL / pvUser2 */);
10145	}
10146
10147
10148	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10149	/**
10150	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
10151	* nested-guest using hardware-assisted VMX.
10152	*
10153	* @param pVCpu The cross context virtual CPU structure.
10154	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
10155	* @param pVmcsInfoGst The guest VMCS info. object.
10156	*/
10157	static void hmR0VmxMergeMsrBitmapNested(PCVMCPU pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
10158	{
10159	uint32_t const cbMsrBitmap = X86_PAGE_4K_SIZE;
10160	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
10161	Assert(pu64MsrBitmap);
10162
10163	/*
10164	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
10165	* MSR that is intercepted by the guest is also intercepted while executing the
10166	* nested-guest using hardware-assisted VMX.
10167	*
10168	* Note! If the nested-guest is not using an MSR bitmap, ever MSR must cause a
10169	* nested-guest VM-exit even if the outer guest is not intercepting some
10170	* MSRs. We cannot assume the caller has initialized the nested-guest
10171	* MSR bitmap in this case.
10172	*
10173	* The guest hypervisor may also switch whether it uses MSR bitmaps for
10174	* each VM-entry, hence initializing it once per-VM while setting up the
10175	* nested-guest VMCS is not sufficient.
10176	*/
10177	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10178	if (pVmcsNstGst->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10179	{
10180	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
10181	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
10182	Assert(pu64MsrBitmapNstGst);
10183	Assert(pu64MsrBitmapGst);
10184
10185	uint32_t const cFrags = cbMsrBitmap / sizeof(uint64_t);
10186	for (uint32_t i = 0; i < cFrags; i++)
10187	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
10188	}
10189	else
10190	ASMMemFill32(pu64MsrBitmap, cbMsrBitmap, UINT32_C(0xffffffff));
10191	}
10192
10193
10194	/**
10195	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
10196	* hardware-assisted VMX execution of the nested-guest.
10197	*
10198	* For a guest, we don't modify these controls once we set up the VMCS and hence
10199	* this function is never called.
10200	*
10201	* For nested-guests since the guest hypervisor provides these controls on every
10202	* nested-guest VM-entry and could potentially change them everytime we need to
10203	* merge them before every nested-guest VM-entry.
10204	*
10205	* @returns VBox status code.
10206	* @param pVCpu The cross context virtual CPU structure.
10207	*/
10208	static int hmR0VmxMergeVmcsNested(PVMCPU pVCpu)
10209	{
10210	PVM pVM = pVCpu->CTX_SUFF(pVM);
10211	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
10212	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
10213	Assert(pVmcsNstGst);
10214
10215	/*
10216	* Merge the controls with the requirements of the guest VMCS.
10217	*
10218	* We do not need to validate the nested-guest VMX features specified in the nested-guest
10219	* VMCS with the features supported by the physical CPU as it's already done by the
10220	* VMLAUNCH/VMRESUME instruction emulation.
10221	*
10222	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the guest are
10223	* derived from the VMX features supported by the physical CPU.
10224	*/
10225
10226	/* Pin-based VM-execution controls. */
10227	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
10228
10229	/* Processor-based VM-execution controls. */
10230	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
10231	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
10232	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
10233	\| VMX_PROC_CTLS_USE_TPR_SHADOW
10234	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
10235
10236	/* Secondary processor-based VM-execution controls. */
10237	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
10238	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
10239	\| VMX_PROC_CTLS2_INVPCID
10240	\| VMX_PROC_CTLS2_VMCS_SHADOWING
10241	\| VMX_PROC_CTLS2_RDTSCP
10242	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
10243	\| VMX_PROC_CTLS2_APIC_REG_VIRT
10244	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
10245	\| VMX_PROC_CTLS2_VMFUNC));
10246
10247	/*
10248	* VM-entry controls:
10249	* These controls contains state that depends on the nested-guest state (primarily
10250	* EFER MSR) and is thus not constant between VMLAUNCH/VMRESUME and the nested-guest
10251	* VM-exit. Although the guest hypervisor cannot change it, we need to in order to
10252	* properly continue executing the nested-guest if the EFER MSR changes but does not
10253	* cause a nested-guest VM-exits.
10254	*
10255	* VM-exit controls:
10256	* These controls specify the host state on return. We cannot use the controls from
10257	* the guest hypervisor state as is as it would contain the guest state rather than
10258	* the host state. Since the host state is subject to change (e.g. preemption, trips
10259	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
10260	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
10261	*
10262	* VM-entry MSR-load:
10263	* The guest MSRs from the VM-entry MSR-load area are already loaded into the guest-CPU
10264	* context by the VMLAUNCH/VMRESUME instruction emulation.
10265	*
10266	* VM-exit MSR-store:
10267	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU context
10268	* back into the VM-exit MSR-store area.
10269	*
10270	* VM-exit MSR-load areas:
10271	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence, we
10272	* can entirely ignore what the guest hypervisor wants to load here.
10273	*/
10274
10275	/*
10276	* Exception bitmap.
10277	*
10278	* We could remove #UD from the guest bitmap and merge it with the nested-guest bitmap
10279	* here (and avoid doing anything while exporting nested-guest state), but to keep the
10280	* code more flexible if intercepting exceptions become more dynamic in the future we do
10281	* it as part of exporting the nested-guest state.
10282	*/
10283	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
10284
10285	/*
10286	* CR0/CR4 guest/host mask.
10287	*
10288	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest must
10289	* cause VM-exits, so we need to merge them here.
10290	*/
10291	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
10292	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
10293
10294	/*
10295	* Page-fault error-code mask and match.
10296	*
10297	* Although we require unrestricted guest execution (and thereby nested-paging) for
10298	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
10299	* normally intercept #PFs, it might intercept them for debugging purposes.
10300	*
10301	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF filters.
10302	* If the outer guest is intercepting #PFs, we must intercept all #PFs.
10303	*/
10304	uint32_t u32XcptPFMask;
10305	uint32_t u32XcptPFMatch;
10306	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
10307	{
10308	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
10309	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
10310	}
10311	else
10312	{
10313	u32XcptPFMask = 0;
10314	u32XcptPFMatch = 0;
10315	}
10316
10317	/*
10318	* Pause-Loop exiting.
10319	*/
10320	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10321	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10322
10323	/*
10324	* I/O Bitmap.
10325	*
10326	* We do not use the I/O bitmap that may be provided by the guest hypervisor as we always
10327	* intercept all I/O port accesses.
10328	*/
10329	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10330
10331	/*
10332	* VMCS shadowing.
10333	*
10334	* We do not yet expose VMCS shadowing to the guest and thus VMCS shadowing should not be
10335	* enabled while executing the nested-guest.
10336	*/
10337	Assert(!(u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING));
10338
10339	/*
10340	* APIC-access page.
10341	*/
10342	RTHCPHYS HCPhysApicAccess;
10343	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10344	{
10345	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS);
10346	RTGCPHYS const GCPhysApicAccess = pVmcsNstGst->u64AddrApicAccess.u;
10347
10348	/** @todo NSTVMX: This is not really correct but currently is required to make
10349	* things work. We need to re-register the page handler when we fallback to
10350	* IEM execution of the nested-guest! */
10351	PGMHandlerPhysicalDeregister(pVM, GCPhysApicAccess);
10352
10353	void *pvPage;
10354	PGMPAGEMAPLOCK PgMapLockApicAccess;
10355	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysApicAccess, &pvPage, &PgMapLockApicAccess);
10356	if (RT_SUCCESS(rc))
10357	{
10358	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysApicAccess, &HCPhysApicAccess);
10359	AssertMsgRCReturn(rc, ("Failed to get host-physical address for APIC-access page at %#RGp\n", GCPhysApicAccess), rc);
10360
10361	/*
10362	* We can release the page lock here because the APIC-access page is never read or
10363	* written to but merely serves as a placeholder in the shadow/nested page tables
10364	* to cause VM-exits or re-direct the access to the virtual-APIC page.
10365	*/
10366	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &PgMapLockApicAccess);
10367	}
10368	else
10369	return rc;
10370	}
10371	else
10372	HCPhysApicAccess = 0;
10373
10374	/*
10375	* Virtual-APIC page and TPR threshold.
10376	*/
10377	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10378	RTHCPHYS HCPhysVirtApic;
10379	uint32_t u32TprThreshold;
10380	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10381	{
10382	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW);
10383
10384	void *pvPage;
10385	RTGCPHYS const GCPhysVirtApic = pVmcsNstGst->u64AddrVirtApic.u;
10386	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysVirtApic, &pvPage, &pVCpu->hm.s.vmx.PgMapLockVirtApic);
10387	AssertMsgRCReturn(rc, ("Failed to get current-context pointer for virtual-APIC page at %#RGp\n", GCPhysVirtApic), rc);
10388
10389	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysVirtApic, &HCPhysVirtApic);
10390	AssertMsgRCReturn(rc, ("Failed to get host-physical address for virtual-APIC page at %#RGp\n", GCPhysVirtApic), rc);
10391	pVCpu->hm.s.vmx.fVirtApicPageLocked = true;
10392
10393	u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10394	}
10395	else
10396	{
10397	HCPhysVirtApic = 0;
10398	u32TprThreshold = 0;
10399
10400	/*
10401	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10402	* used by the guest hypervisor. Preventing MMIO accesses to the physical APIC will
10403	* be taken care of by EPT/shadow paging.
10404	*/
10405	if (pVM->hm.s.fAllow64BitGuests)
10406	{
10407	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10408	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10409	}
10410	}
10411
10412	/*
10413	* Validate basic assumptions.
10414	*/
10415	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10416	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10417	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10418
10419	/*
10420	* Commit it to the nested-guest VMCS.
10421	*/
10422	int rc = VINF_SUCCESS;
10423	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10424	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10425	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10426	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10427	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10428	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10429	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10430	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10431	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10432	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10433	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10434	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10435	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10436	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10437	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10438	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10439	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10440	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10441	{
10442	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10443	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10444	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10445	}
10446	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10447	{
10448	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10449	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
10450	}
10451	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10452	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
10453	AssertRCReturn(rc, rc);
10454
10455	/*
10456	* Update the nested-guest VMCS cache.
10457	*/
10458	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10459	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10460	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10461	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10462	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10463	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10464	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10465	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10466	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10467
10468	/*
10469	* We need to flush the TLB if we are switching the APIC-access page address.
10470	* See Intel spec. 28.3.3.4 "Guidelines for Use of the INVEPT Instruction".
10471	*/
10472	if (u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10473	pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = true;
10474
10475	/*
10476	* MSR bitmap.
10477	*
10478	* The MSR bitmap address has already been initialized while setting up the nested-guest
10479	* VMCS, here we need to merge the MSR bitmaps.
10480	*/
10481	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10482	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10483
10484	return VINF_SUCCESS;
10485	}
10486	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10487
10488
10489	/**
10490	* Does the preparations before executing guest code in VT-x.
10491	*
10492	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10493	* recompiler/IEM. We must be cautious what we do here regarding committing
10494	* guest-state information into the VMCS assuming we assuredly execute the
10495	* guest in VT-x mode.
10496	*
10497	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10498	* the common-state (TRPM/forceflags), we must undo those changes so that the
10499	* recompiler/IEM can (and should) use them when it resumes guest execution.
10500	* Otherwise such operations must be done when we can no longer exit to ring-3.
10501	*
10502	* @returns Strict VBox status code (i.e. informational status codes too).
10503	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10504	* have been disabled.
10505	* @retval VINF_VMX_VMEXIT if a nested-guest VM-exit occurs (e.g., while evaluating
10506	* pending events).
10507	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10508	* double-fault into the guest.
10509	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10510	* dispatched directly.
10511	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10512	*
10513	* @param pVCpu The cross context virtual CPU structure.
10514	* @param pVmxTransient The VMX-transient structure.
10515	* @param fStepping Whether we are single-stepping the guest in the
10516	* hypervisor debugger. Makes us ignore some of the reasons
10517	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10518	* if event dispatching took place.
10519	*/
10520	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
10521	{
10522	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10523
10524	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
10525	if (pVmxTransient->fIsNestedGuest)
10526	{
10527	RT_NOREF2(pVCpu, fStepping);
10528	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
10529	return VINF_EM_RESCHEDULE_REM;
10530	}
10531	#endif
10532
10533	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
10534	PGMRZDynMapFlushAutoSet(pVCpu);
10535	#endif
10536
10537	/*
10538	* Check and process force flag actions, some of which might require us to go back to ring-3.
10539	*/
10540	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
10541	if (rcStrict == VINF_SUCCESS)
10542	{
10543	/* FFs don't get set all the time. */
10544	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10545	if ( pVmxTransient->fIsNestedGuest
10546	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
10547	{
10548	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
10549	return VINF_VMX_VMEXIT;
10550	}
10551	#endif
10552	}
10553	else
10554	return rcStrict;
10555
10556	/*
10557	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
10558	*/
10559	/** @todo Doing this from ring-3 after VM setup phase causes a
10560	* VERR_IOM_MMIO_RANGE_NOT_FOUND guru while booting Visa 64 SMP VM. No
10561	* idea why atm. */
10562	PVM pVM = pVCpu->CTX_SUFF(pVM);
10563	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
10564	&& (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10565	&& PDMHasApic(pVM))
10566	{
10567	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
10568	AssertRCReturn(rc, rc);
10569	}
10570
10571	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10572	/*
10573	* Merge guest VMCS controls with the nested-guest VMCS controls.
10574	*
10575	* Even if we have not executed the guest prior to this (e.g. when resuming from a
10576	* saved state), we should be okay with merging controls as we initialize the
10577	* guest VMCS controls as part of VM setup phase.
10578	*/
10579	if ( pVmxTransient->fIsNestedGuest
10580	&& !pVCpu->hm.s.vmx.fMergedNstGstCtls)
10581	{
10582	int rc = hmR0VmxMergeVmcsNested(pVCpu);
10583	AssertRCReturn(rc, rc);
10584	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
10585	}
10586	#endif
10587
10588	/*
10589	* Evaluate events to be injected into the guest.
10590	*
10591	* Events in TRPM can be injected without inspecting the guest state.
10592	* If any new events (interrupts/NMI) are pending currently, we try to set up the
10593	* guest to cause a VM-exit the next time they are ready to receive the event.
10594	*
10595	* With nested-guests, evaluating pending events may cause VM-exits.
10596	*/
10597	if (TRPMHasTrap(pVCpu))
10598	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
10599
10600	uint32_t fIntrState;
10601	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
10602
10603	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10604	/*
10605	* While evaluating pending events if something failed (unlikely) or if we were
10606	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
10607	*/
10608	if (rcStrict != VINF_SUCCESS)
10609	return rcStrict;
10610	if ( pVmxTransient->fIsNestedGuest
10611	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
10612	{
10613	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
10614	return VINF_VMX_VMEXIT;
10615	}
10616	#else
10617	Assert(rcStrict == VINF_SUCCESS);
10618	#endif
10619
10620	/*
10621	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
10622	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
10623	* also result in triple-faulting the VM.
10624	*
10625	* With nested-guests, the above does not apply since unrestricted guest execution is a
10626	* requirement. Regardless, we do this here to avoid duplicating code elsewhere.
10627	*/
10628	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
10629	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10630	{ /* likely */ }
10631	else
10632	{
10633	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
10634	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10635	return rcStrict;
10636	}
10637
10638	/*
10639	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
10640	* import CR3 themselves. We will need to update them here, as even as late as the above
10641	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
10642	* the below force flags to be set.
10643	*/
10644	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
10645	{
10646	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
10647	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
10648	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
10649	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
10650	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10651	}
10652	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
10653	{
10654	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
10655	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10656	}
10657
10658	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10659	/* Paranoia. */
10660	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10661	#endif
10662
10663	/*
10664	* No longjmps to ring-3 from this point on!!!
10665	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
10666	* This also disables flushing of the R0-logger instance (if any).
10667	*/
10668	VMMRZCallRing3Disable(pVCpu);
10669
10670	/*
10671	* Export the guest state bits.
10672	*
10673	* We cannot perform longjmps while loading the guest state because we do not preserve the
10674	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
10675	* CPU migration.
10676	*
10677	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
10678	* registers. Hence, exporting of the guest state needs to be done -after- injection of events.
10679	*/
10680	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
10681	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10682	{ /* likely */ }
10683	else
10684	{
10685	VMMRZCallRing3Enable(pVCpu);
10686	return rcStrict;
10687	}
10688
10689	/*
10690	* We disable interrupts so that we don't miss any interrupts that would flag preemption
10691	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
10692	* preemption disabled for a while. Since this is purely to aid the
10693	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
10694	* disable interrupt on NT.
10695	*
10696	* We need to check for force-flags that could've possible been altered since we last
10697	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
10698	* see @bugref{6398}).
10699	*
10700	* We also check a couple of other force-flags as a last opportunity to get the EMT back
10701	* to ring-3 before executing guest code.
10702	*/
10703	pVmxTransient->fEFlags = ASMIntDisableFlags();
10704
10705	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
10706	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
10707	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
10708	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
10709	{
10710	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
10711	{
10712	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10713	/*
10714	* If we are executing a nested-guest make sure that we should intercept subsequent
10715	* events. The one we are injecting might be part of VM-entry. This is mainly to keep
10716	* the VM-exit instruction emulation happy.
10717	*/
10718	if (pVmxTransient->fIsNestedGuest)
10719	pVCpu->cpum.GstCtx.hwvirt.vmx.fInterceptEvents = true;
10720	#endif
10721
10722	/*
10723	* We've injected any pending events. This is really the point of no return (to ring-3).
10724	*
10725	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
10726	* returns from this function, so do -not- enable them here.
10727	*/
10728	pVCpu->hm.s.Event.fPending = false;
10729	return VINF_SUCCESS;
10730	}
10731
10732	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
10733	rcStrict = VINF_EM_RAW_INTERRUPT;
10734	}
10735	else
10736	{
10737	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
10738	rcStrict = VINF_EM_RAW_TO_R3;
10739	}
10740
10741	ASMSetFlags(pVmxTransient->fEFlags);
10742	VMMRZCallRing3Enable(pVCpu);
10743
10744	return rcStrict;
10745	}
10746
10747
10748	/**
10749	* Final preparations before executing guest code using hardware-assisted VMX.
10750	*
10751	* We can no longer get preempted to a different host CPU and there are no returns
10752	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
10753	* failures), this function is not intended to fail sans unrecoverable hardware
10754	* errors.
10755	*
10756	* @param pVCpu The cross context virtual CPU structure.
10757	* @param pVmxTransient The VMX-transient structure.
10758	*
10759	* @remarks Called with preemption disabled.
10760	* @remarks No-long-jump zone!!!
10761	*/
10762	static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10763	{
10764	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10765	Assert(VMMR0IsLogFlushDisabled(pVCpu));
10766	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
10767	Assert(!pVCpu->hm.s.Event.fPending);
10768
10769	/*
10770	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
10771	*/
10772	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10773	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
10774
10775	PVM pVM = pVCpu->CTX_SUFF(pVM);
10776	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10777	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
10778	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
10779
10780	if (!CPUMIsGuestFPUStateActive(pVCpu))
10781	{
10782	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10783	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
10784	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
10785	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10786	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
10787	}
10788
10789	/*
10790	* Re-export the host state bits as we may've been preempted (only happens when
10791	* thread-context hooks are used or when the VM start function changes) or if
10792	* the host CR0 is modified while loading the guest FPU state above.
10793	*
10794	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
10795	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
10796	* see @bugref{8432}.
10797	*
10798	* This may also happen when switching to/from a nested-guest VMCS without leaving
10799	* ring-0.
10800	*/
10801	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
10802	{
10803	hmR0VmxExportHostState(pVCpu);
10804	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportHostState);
10805	}
10806	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
10807
10808	/*
10809	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
10810	*/
10811	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
10812	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
10813	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
10814
10815	/*
10816	* Store status of the shared guest/host debug state at the time of VM-entry.
10817	*/
10818	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
10819	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
10820	{
10821	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
10822	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
10823	}
10824	else
10825	#endif
10826	{
10827	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
10828	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
10829	}
10830
10831	/*
10832	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
10833	* more than one conditional check. The post-run side of our code shall determine
10834	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
10835	*/
10836	if (pVmcsInfo->pbVirtApic)
10837	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
10838
10839	/*
10840	* Update the host MSRs values in the VM-exit MSR-load area.
10841	*/
10842	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
10843	{
10844	if (pVmcsInfo->cExitMsrLoad > 0)
10845	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
10846	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
10847	}
10848
10849	/*
10850	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
10851	* VMX-preemption timer based on the next virtual sync clock deadline.
10852	*/
10853	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
10854	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
10855	{
10856	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
10857	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
10858	}
10859
10860	/* Record statistics of how often we use TSC offsetting as opposed to intercepting RDTSC/P. */
10861	bool const fIsRdtscIntercepted = RT_BOOL(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT);
10862	if (!fIsRdtscIntercepted)
10863	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
10864	else
10865	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
10866
10867	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
10868	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
10869	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
10870	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
10871	pVmcsInfo->idHostCpu = idCurrentCpu; /* Update the CPU for which we updated host-state in this VMCS. */
10872
10873	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
10874
10875	TMNotifyStartOfExecution(pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
10876	as we're about to start executing the guest . */
10877
10878	/*
10879	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
10880	*
10881	* This is done this late as updating the TSC offsetting/preemption timer above
10882	* figures out if we can skip intercepting RDTSCP by calculating the number of
10883	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
10884	*/
10885	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
10886	&& !fIsRdtscIntercepted)
10887	{
10888	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
10889
10890	/* NB: Because we call hmR0VmxAddAutoLoadStoreMsr with fUpdateHostMsr=true,
10891	it's safe even after hmR0VmxUpdateAutoLoadHostMsrs has already been done. */
10892	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
10893	true /* fSetReadWrite /, true / fUpdateHostMsr */);
10894	AssertRC(rc);
10895	Assert(!pVmxTransient->fRemoveTscAuxMsr);
10896	pVmxTransient->fRemoveTscAuxMsr = true;
10897	}
10898
10899	#ifdef VBOX_STRICT
10900	Assert(pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs);
10901	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
10902	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
10903	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest));
10904	#endif
10905
10906	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
10907	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
10908	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs,
10909	* see @bugref{9180#c54}. */
10910	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
10911	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
10912	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
10913	#endif
10914	}
10915
10916
10917	/**
10918	* First C routine invoked after running guest code using hardware-assisted VMX.
10919	*
10920	* @param pVCpu The cross context virtual CPU structure.
10921	* @param pVmxTransient The VMX-transient structure.
10922	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
10923	*
10924	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
10925	*
10926	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
10927	* unconditionally when it is safe to do so.
10928	*/
10929	static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
10930	{
10931	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
10932
10933	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
10934	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
10935	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
10936	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
10937	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
10938	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
10939
10940	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10941	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10942	{
10943	uint64_t uGstTsc;
10944	if (!pVmxTransient->fIsNestedGuest)
10945	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10946	else
10947	{
10948	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10949	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
10950	}
10951	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
10952	}
10953
10954	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
10955	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
10956	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10957
10958	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
10959	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10960	#ifdef VBOX_STRICT
10961	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
10962	#endif
10963	Assert(!ASMIntAreEnabled());
10964	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
10965	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10966
10967	#ifdef HMVMX_ALWAYS_CLEAN_TRANSIENT
10968	/*
10969	* Clean all the VMCS fields in the transient structure before reading
10970	* anything from the VMCS.
10971	*/
10972	pVmxTransient->uExitReason = 0;
10973	pVmxTransient->uExitIntErrorCode = 0;
10974	pVmxTransient->uExitQual = 0;
10975	pVmxTransient->uGuestLinearAddr = 0;
10976	pVmxTransient->uExitIntInfo = 0;
10977	pVmxTransient->cbInstr = 0;
10978	pVmxTransient->ExitInstrInfo.u = 0;
10979	pVmxTransient->uEntryIntInfo = 0;
10980	pVmxTransient->uEntryXcptErrorCode = 0;
10981	pVmxTransient->cbEntryInstr = 0;
10982	pVmxTransient->uIdtVectoringInfo = 0;
10983	pVmxTransient->uIdtVectoringErrorCode = 0;
10984	#endif
10985
10986	/*
10987	* Save the basic VM-exit reason and check if the VM-entry failed.
10988	* See Intel spec. 24.9.1 "Basic VM-exit Information".
10989	*/
10990	uint32_t uExitReason;
10991	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
10992	AssertRC(rc);
10993	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
10994	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
10995
10996	/*
10997	* Log the VM-exit before logging anything else as otherwise it might be a
10998	* tad confusing what happens before and after the world-switch.
10999	*/
11000	HMVMX_LOG_EXIT(pVCpu, uExitReason);
11001
11002	/*
11003	* Remove the TSC_AUX MSR from the auto-load/store MSR area and reset any MSR
11004	* bitmap permissions, if it was added before VM-entry.
11005	*/
11006	if (pVmxTransient->fRemoveTscAuxMsr)
11007	{
11008	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
11009	pVmxTransient->fRemoveTscAuxMsr = false;
11010	}
11011
11012	/*
11013	* Check if VMLAUNCH/VMRESUME succeeded.
11014	* If this failed, we cause a guru meditation and cease further execution.
11015	*
11016	* However, if we are executing a nested-guest we might fail if we use the
11017	* fast path rather than fully emulating VMLAUNCH/VMRESUME instruction in IEM.
11018	*/
11019	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
11020	{
11021	/*
11022	* Update the VM-exit history array here even if the VM-entry failed due to:
11023	* - Invalid guest state.
11024	* - MSR loading.
11025	* - Machine-check event.
11026	*
11027	* In any of the above cases we will still have a "valid" VM-exit reason
11028	* despite @a fVMEntryFailed being false.
11029	*
11030	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
11031	*
11032	* Note! We don't have CS or RIP at this point. Will probably address that later
11033	* by amending the history entry added here.
11034	*/
11035	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
11036	UINT64_MAX, uHostTsc);
11037
11038	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
11039	{
11040	VMMRZCallRing3Enable(pVCpu);
11041
11042	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
11043	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
11044
11045	#ifdef HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
11046	rc = hmR0VmxReadAllRoFieldsVmcs(pVCpu, pVmxTransient);
11047	AssertRC(rc);
11048	#endif
11049	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
11050	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11051	AssertRC(rc);
11052	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
11053	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
11054	AssertRC(rc);
11055	#else
11056	/*
11057	* Import the guest-interruptibility state always as we need it while evaluating
11058	* injecting events on re-entry.
11059	*
11060	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
11061	* checking for real-mode while exporting the state because all bits that cause
11062	* mode changes wrt CR0 are intercepted.
11063	*/
11064	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
11065	AssertRC(rc);
11066	#endif
11067
11068	/*
11069	* Sync the TPR shadow with our APIC state.
11070	*/
11071	if ( !pVmxTransient->fIsNestedGuest
11072	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
11073	{
11074	Assert(pVmcsInfo->pbVirtApic);
11075	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
11076	{
11077	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
11078	AssertRC(rc);
11079	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11080	}
11081	}
11082
11083	Assert(VMMRZCallRing3IsEnabled(pVCpu));
11084	return;
11085	}
11086	}
11087	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11088	else if (pVmxTransient->fIsNestedGuest)
11089	AssertMsgFailed(("VMLAUNCH/VMRESUME failed but shouldn't happen when VMLAUNCH/VMRESUME was emulated in IEM!\n"));
11090	#endif
11091	else
11092	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
11093
11094	VMMRZCallRing3Enable(pVCpu);
11095	}
11096
11097
11098	/**
11099	* Runs the guest code using hardware-assisted VMX the normal way.
11100	*
11101	* @returns VBox status code.
11102	* @param pVCpu The cross context virtual CPU structure.
11103	* @param pcLoops Pointer to the number of executed loops.
11104	*/
11105	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, uint32_t *pcLoops)
11106	{
11107	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11108	Assert(pcLoops);
11109	Assert(*pcLoops <= cMaxResumeLoops);
11110	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11111
11112	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11113	/*
11114	* Switch to the guest VMCS as we may have transitioned from executing the nested-guest
11115	* without leaving ring-0. Otherwise, if we came from ring-3 we would have loaded the
11116	* guest VMCS while entering the VMX ring-0 session.
11117	*/
11118	if (pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
11119	{
11120	int rc = hmR0VmxSwitchToGstOrNstGstVmcs(pVCpu, false /* fSwitchToNstGstVmcs */);
11121	if (RT_SUCCESS(rc))
11122	{ /* likely */ }
11123	else
11124	{
11125	LogRelFunc(("Failed to switch to the guest VMCS. rc=%Rrc\n", rc));
11126	return rc;
11127	}
11128	}
11129	#endif
11130
11131	VMXTRANSIENT VmxTransient;
11132	RT_ZERO(VmxTransient);
11133	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11134
11135	/* Paranoia. */
11136	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
11137
11138	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11139	for (;;)
11140	{
11141	Assert(!HMR0SuspendPending());
11142	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11143	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11144
11145	/*
11146	* Preparatory work for running nested-guest code, this may force us to
11147	* return to ring-3.
11148	*
11149	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11150	*/
11151	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
11152	if (rcStrict != VINF_SUCCESS)
11153	break;
11154
11155	/* Interrupts are disabled at this point! */
11156	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11157	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11158	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11159	/* Interrupts are re-enabled at this point! */
11160
11161	/*
11162	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
11163	*/
11164	if (RT_SUCCESS(rcRun))
11165	{ /* very likely */ }
11166	else
11167	{
11168	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11169	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11170	return rcRun;
11171	}
11172
11173	/*
11174	* Profile the VM-exit.
11175	*/
11176	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11177	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11178	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11179	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11180	HMVMX_START_EXIT_DISPATCH_PROF();
11181
11182	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11183
11184	/*
11185	* Handle the VM-exit.
11186	*/
11187	#ifdef HMVMX_USE_FUNCTION_TABLE
11188	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
11189	#else
11190	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
11191	#endif
11192	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11193	if (rcStrict == VINF_SUCCESS)
11194	{
11195	if (++(*pcLoops) <= cMaxResumeLoops)
11196	continue;
11197	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11198	rcStrict = VINF_EM_RAW_INTERRUPT;
11199	}
11200	break;
11201	}
11202
11203	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11204	return rcStrict;
11205	}
11206
11207
11208	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
11209	/**
11210	* Runs the nested-guest code using hardware-assisted VMX.
11211	*
11212	* @returns VBox status code.
11213	* @param pVCpu The cross context virtual CPU structure.
11214	* @param pcLoops Pointer to the number of executed loops.
11215	*
11216	* @sa hmR0VmxRunGuestCodeNormal.
11217	*/
11218	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPU pVCpu, uint32_t *pcLoops)
11219	{
11220	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11221	Assert(pcLoops);
11222	Assert(*pcLoops <= cMaxResumeLoops);
11223	Assert(CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
11224
11225	/*
11226	* Switch to the nested-guest VMCS as we may have transitioned from executing the
11227	* guest without leaving ring-0. Otherwise, if we came from ring-3 we would have
11228	* loaded the nested-guest VMCS while entering the VMX ring-0 session.
11229	*/
11230	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
11231	{
11232	int rc = hmR0VmxSwitchToGstOrNstGstVmcs(pVCpu, true /* fSwitchToNstGstVmcs */);
11233	if (RT_SUCCESS(rc))
11234	{ /* likely */ }
11235	else
11236	{
11237	LogRelFunc(("Failed to switch to the nested-guest VMCS. rc=%Rrc\n", rc));
11238	return rc;
11239	}
11240	}
11241
11242	VMXTRANSIENT VmxTransient;
11243	RT_ZERO(VmxTransient);
11244	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11245	VmxTransient.fIsNestedGuest = true;
11246
11247	/* Paranoia. */
11248	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfoNstGst);
11249
11250	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11251	for (;;)
11252	{
11253	Assert(!HMR0SuspendPending());
11254	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11255	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11256
11257	/*
11258	* Preparatory work for running guest code, this may force us to
11259	* return to ring-3.
11260	*
11261	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11262	*/
11263	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
11264	if (rcStrict != VINF_SUCCESS)
11265	break;
11266
11267	/* Interrupts are disabled at this point! */
11268	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11269	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
11270	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
11271	/* Interrupts are re-enabled at this point! */
11272
11273	/*
11274	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
11275	*/
11276	if (RT_SUCCESS(rcRun))
11277	{ /* very likely */ }
11278	else
11279	{
11280	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
11281	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
11282	return rcRun;
11283	}
11284
11285	/*
11286	* Profile the VM-exit.
11287	*/
11288	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
11289	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
11290	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitAll);
11291	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
11292	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
11293	HMVMX_START_EXIT_DISPATCH_PROF();
11294
11295	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
11296
11297	/*
11298	* Handle the VM-exit.
11299	*/
11300	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
11301	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
11302	if (rcStrict == VINF_SUCCESS)
11303	{
11304	if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
11305	{
11306	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
11307	rcStrict = VINF_VMX_VMEXIT;
11308	}
11309	else
11310	{
11311	if (++(*pcLoops) <= cMaxResumeLoops)
11312	continue;
11313	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
11314	rcStrict = VINF_EM_RAW_INTERRUPT;
11315	}
11316	}
11317	else
11318	Assert(rcStrict != VINF_VMX_VMEXIT);
11319	break;
11320	}
11321
11322	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
11323	return rcStrict;
11324	}
11325	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
11326
11327
11328	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
11329	* probes.
11330	*
11331	* The following few functions and associated structure contains the bloat
11332	* necessary for providing detailed debug events and dtrace probes as well as
11333	* reliable host side single stepping. This works on the principle of
11334	* "subclassing" the normal execution loop and workers. We replace the loop
11335	* method completely and override selected helpers to add necessary adjustments
11336	* to their core operation.
11337	*
11338	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
11339	* any performance for debug and analysis features.
11340	*
11341	* @{
11342	*/
11343
11344	/**
11345	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
11346	* the debug run loop.
11347	*/
11348	typedef struct VMXRUNDBGSTATE
11349	{
11350	/** The RIP we started executing at. This is for detecting that we stepped. */
11351	uint64_t uRipStart;
11352	/** The CS we started executing with. */
11353	uint16_t uCsStart;
11354
11355	/** Whether we've actually modified the 1st execution control field. */
11356	bool fModifiedProcCtls : 1;
11357	/** Whether we've actually modified the 2nd execution control field. */
11358	bool fModifiedProcCtls2 : 1;
11359	/** Whether we've actually modified the exception bitmap. */
11360	bool fModifiedXcptBitmap : 1;
11361
11362	/** We desire the modified the CR0 mask to be cleared. */
11363	bool fClearCr0Mask : 1;
11364	/** We desire the modified the CR4 mask to be cleared. */
11365	bool fClearCr4Mask : 1;
11366	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
11367	uint32_t fCpe1Extra;
11368	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
11369	uint32_t fCpe1Unwanted;
11370	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
11371	uint32_t fCpe2Extra;
11372	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
11373	uint32_t bmXcptExtra;
11374	/** The sequence number of the Dtrace provider settings the state was
11375	* configured against. */
11376	uint32_t uDtraceSettingsSeqNo;
11377	/** VM-exits to check (one bit per VM-exit). */
11378	uint32_t bmExitsToCheck[3];
11379
11380	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
11381	uint32_t fProcCtlsInitial;
11382	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
11383	uint32_t fProcCtls2Initial;
11384	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
11385	uint32_t bmXcptInitial;
11386	} VMXRUNDBGSTATE;
11387	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
11388	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
11389
11390
11391	/**
11392	* Initializes the VMXRUNDBGSTATE structure.
11393	*
11394	* @param pVCpu The cross context virtual CPU structure of the
11395	* calling EMT.
11396	* @param pVmxTransient The VMX-transient structure.
11397	* @param pDbgState The debug state to initialize.
11398	*/
11399	static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11400	{
11401	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
11402	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
11403
11404	pDbgState->fModifiedProcCtls = false;
11405	pDbgState->fModifiedProcCtls2 = false;
11406	pDbgState->fModifiedXcptBitmap = false;
11407	pDbgState->fClearCr0Mask = false;
11408	pDbgState->fClearCr4Mask = false;
11409	pDbgState->fCpe1Extra = 0;
11410	pDbgState->fCpe1Unwanted = 0;
11411	pDbgState->fCpe2Extra = 0;
11412	pDbgState->bmXcptExtra = 0;
11413	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
11414	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
11415	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
11416	}
11417
11418
11419	/**
11420	* Updates the VMSC fields with changes requested by @a pDbgState.
11421	*
11422	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
11423	* immediately before executing guest code, i.e. when interrupts are disabled.
11424	* We don't check status codes here as we cannot easily assert or return in the
11425	* latter case.
11426	*
11427	* @param pVCpu The cross context virtual CPU structure.
11428	* @param pVmxTransient The VMX-transient structure.
11429	* @param pDbgState The debug state.
11430	*/
11431	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11432	{
11433	/*
11434	* Ensure desired flags in VMCS control fields are set.
11435	* (Ignoring write failure here, as we're committed and it's just debug extras.)
11436	*
11437	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
11438	* there should be no stale data in pCtx at this point.
11439	*/
11440	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11441	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11442	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11443	{
11444	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11445	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11446	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11447	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11448	pDbgState->fModifiedProcCtls = true;
11449	}
11450
11451	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11452	{
11453	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11454	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11455	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11456	pDbgState->fModifiedProcCtls2 = true;
11457	}
11458
11459	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11460	{
11461	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11462	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11463	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11464	pDbgState->fModifiedXcptBitmap = true;
11465	}
11466
11467	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11468	{
11469	pVmcsInfo->u64Cr0Mask = 0;
11470	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, 0);
11471	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11472	}
11473
11474	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11475	{
11476	pVmcsInfo->u64Cr4Mask = 0;
11477	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, 0);
11478	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11479	}
11480
11481	NOREF(pVCpu);
11482	}
11483
11484
11485	/**
11486	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11487	* re-entry next time around.
11488	*
11489	* @returns Strict VBox status code (i.e. informational status codes too).
11490	* @param pVCpu The cross context virtual CPU structure.
11491	* @param pVmxTransient The VMX-transient structure.
11492	* @param pDbgState The debug state.
11493	* @param rcStrict The return code from executing the guest using single
11494	* stepping.
11495	*/
11496	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11497	VBOXSTRICTRC rcStrict)
11498	{
11499	/*
11500	* Restore VM-exit control settings as we may not reenter this function the
11501	* next time around.
11502	*/
11503	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11504
11505	/* We reload the initial value, trigger what we can of recalculations the
11506	next time around. From the looks of things, that's all that's required atm. */
11507	if (pDbgState->fModifiedProcCtls)
11508	{
11509	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11510	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11511	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11512	AssertRCReturn(rc2, rc2);
11513	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11514	}
11515
11516	/* We're currently the only ones messing with this one, so just restore the
11517	cached value and reload the field. */
11518	if ( pDbgState->fModifiedProcCtls2
11519	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11520	{
11521	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
11522	AssertRCReturn(rc2, rc2);
11523	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
11524	}
11525
11526	/* If we've modified the exception bitmap, we restore it and trigger
11527	reloading and partial recalculation the next time around. */
11528	if (pDbgState->fModifiedXcptBitmap)
11529	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
11530
11531	return rcStrict;
11532	}
11533
11534
11535	/**
11536	* Configures VM-exit controls for current DBGF and DTrace settings.
11537	*
11538	* This updates @a pDbgState and the VMCS execution control fields to reflect
11539	* the necessary VM-exits demanded by DBGF and DTrace.
11540	*
11541	* @param pVCpu The cross context virtual CPU structure.
11542	* @param pVmxTransient The VMX-transient structure. May update
11543	* fUpdatedTscOffsettingAndPreemptTimer.
11544	* @param pDbgState The debug state.
11545	*/
11546	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11547	{
11548	/*
11549	* Take down the dtrace serial number so we can spot changes.
11550	*/
11551	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
11552	ASMCompilerBarrier();
11553
11554	/*
11555	* We'll rebuild most of the middle block of data members (holding the
11556	* current settings) as we go along here, so start by clearing it all.
11557	*/
11558	pDbgState->bmXcptExtra = 0;
11559	pDbgState->fCpe1Extra = 0;
11560	pDbgState->fCpe1Unwanted = 0;
11561	pDbgState->fCpe2Extra = 0;
11562	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
11563	pDbgState->bmExitsToCheck[i] = 0;
11564
11565	/*
11566	* Software interrupts (INT XXh) - no idea how to trigger these...
11567	*/
11568	PVM pVM = pVCpu->CTX_SUFF(pVM);
11569	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
11570	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
11571	{
11572	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11573	}
11574
11575	/*
11576	* INT3 breakpoints - triggered by #BP exceptions.
11577	*/
11578	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
11579	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11580
11581	/*
11582	* Exception bitmap and XCPT events+probes.
11583	*/
11584	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
11585	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
11586	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
11587
11588	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
11589	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
11590	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11591	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
11592	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
11593	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
11594	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
11595	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
11596	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
11597	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
11598	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
11599	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
11600	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
11601	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
11602	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
11603	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
11604	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
11605	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
11606
11607	if (pDbgState->bmXcptExtra)
11608	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11609
11610	/*
11611	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
11612	*
11613	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
11614	* So, when adding/changing/removing please don't forget to update it.
11615	*
11616	* Some of the macros are picking up local variables to save horizontal space,
11617	* (being able to see it in a table is the lesser evil here).
11618	*/
11619	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
11620	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
11621	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
11622	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
11623	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11624	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11625	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11626	} else do { } while (0)
11627	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
11628	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11629	{ \
11630	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
11631	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11632	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11633	} else do { } while (0)
11634	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
11635	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11636	{ \
11637	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
11638	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11639	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11640	} else do { } while (0)
11641	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
11642	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11643	{ \
11644	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
11645	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11646	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11647	} else do { } while (0)
11648
11649	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
11650	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
11651	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
11652	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
11653	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
11654
11655	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
11656	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
11657	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
11658	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
11659	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
11660	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
11661	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
11662	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
11663	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
11664	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
11665	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
11666	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
11667	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
11668	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
11669	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
11670	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
11671	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
11672	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
11673	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
11674	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
11675	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
11676	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
11677	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
11678	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
11679	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
11680	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
11681	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
11682	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
11683	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
11684	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
11685	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
11686	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
11687	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
11688	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
11689	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
11690	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
11691
11692	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
11693	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11694	{
11695	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
11696	\| CPUMCTX_EXTRN_APIC_TPR);
11697	AssertRC(rc);
11698
11699	#if 0 /** @todo fix me */
11700	pDbgState->fClearCr0Mask = true;
11701	pDbgState->fClearCr4Mask = true;
11702	#endif
11703	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
11704	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
11705	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11706	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
11707	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
11708	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
11709	require clearing here and in the loop if we start using it. */
11710	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
11711	}
11712	else
11713	{
11714	if (pDbgState->fClearCr0Mask)
11715	{
11716	pDbgState->fClearCr0Mask = false;
11717	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11718	}
11719	if (pDbgState->fClearCr4Mask)
11720	{
11721	pDbgState->fClearCr4Mask = false;
11722	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11723	}
11724	}
11725	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
11726	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
11727
11728	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
11729	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
11730	{
11731	/** @todo later, need to fix handler as it assumes this won't usually happen. */
11732	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
11733	}
11734	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
11735	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
11736
11737	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
11738	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
11739	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
11740	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
11741	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
11742	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
11743	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
11744	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
11745	#if 0 /** @todo too slow, fix handler. */
11746	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
11747	#endif
11748	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
11749
11750	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
11751	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
11752	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
11753	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
11754	{
11755	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11756	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
11757	}
11758	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11759	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11760	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11761	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11762
11763	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
11764	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
11765	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
11766	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
11767	{
11768	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11769	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
11770	}
11771	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
11772	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
11773	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
11774	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
11775
11776	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
11777	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
11778	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
11779	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
11780	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
11781	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
11782	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
11783	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
11784	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
11785	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
11786	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
11787	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
11788	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
11789	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
11790	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
11791	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
11792	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
11793	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
11794	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
11795	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
11796	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
11797	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
11798
11799	#undef IS_EITHER_ENABLED
11800	#undef SET_ONLY_XBM_IF_EITHER_EN
11801	#undef SET_CPE1_XBM_IF_EITHER_EN
11802	#undef SET_CPEU_XBM_IF_EITHER_EN
11803	#undef SET_CPE2_XBM_IF_EITHER_EN
11804
11805	/*
11806	* Sanitize the control stuff.
11807	*/
11808	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
11809	if (pDbgState->fCpe2Extra)
11810	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
11811	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
11812	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
11813	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
11814	{
11815	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
11816	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11817	}
11818
11819	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
11820	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
11821	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
11822	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
11823	}
11824
11825
11826	/**
11827	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
11828	* appropriate.
11829	*
11830	* The caller has checked the VM-exit against the
11831	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
11832	* already, so we don't have to do that either.
11833	*
11834	* @returns Strict VBox status code (i.e. informational status codes too).
11835	* @param pVCpu The cross context virtual CPU structure.
11836	* @param pVmxTransient The VMX-transient structure.
11837	* @param uExitReason The VM-exit reason.
11838	*
11839	* @remarks The name of this function is displayed by dtrace, so keep it short
11840	* and to the point. No longer than 33 chars long, please.
11841	*/
11842	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
11843	{
11844	/*
11845	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
11846	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
11847	*
11848	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
11849	* does. Must add/change/remove both places. Same ordering, please.
11850	*
11851	* Added/removed events must also be reflected in the next section
11852	* where we dispatch dtrace events.
11853	*/
11854	bool fDtrace1 = false;
11855	bool fDtrace2 = false;
11856	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
11857	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
11858	uint32_t uEventArg = 0;
11859	#define SET_EXIT(a_EventSubName) \
11860	do { \
11861	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11862	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11863	} while (0)
11864	#define SET_BOTH(a_EventSubName) \
11865	do { \
11866	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
11867	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11868	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
11869	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11870	} while (0)
11871	switch (uExitReason)
11872	{
11873	case VMX_EXIT_MTF:
11874	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11875
11876	case VMX_EXIT_XCPT_OR_NMI:
11877	{
11878	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11879	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
11880	{
11881	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11882	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
11883	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
11884	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
11885	{
11886	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
11887	{
11888	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11889	uEventArg = pVmxTransient->uExitIntErrorCode;
11890	}
11891	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
11892	switch (enmEvent1)
11893	{
11894	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
11895	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
11896	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
11897	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
11898	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
11899	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
11900	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
11901	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
11902	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
11903	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
11904	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
11905	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
11906	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
11907	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
11908	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
11909	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
11910	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
11911	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
11912	default: break;
11913	}
11914	}
11915	else
11916	AssertFailed();
11917	break;
11918
11919	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
11920	uEventArg = idxVector;
11921	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
11922	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
11923	break;
11924	}
11925	break;
11926	}
11927
11928	case VMX_EXIT_TRIPLE_FAULT:
11929	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
11930	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
11931	break;
11932	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
11933	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
11934	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
11935	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
11936	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
11937
11938	/* Instruction specific VM-exits: */
11939	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
11940	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
11941	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
11942	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
11943	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
11944	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
11945	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
11946	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
11947	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
11948	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
11949	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
11950	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
11951	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
11952	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
11953	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
11954	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
11955	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
11956	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
11957	case VMX_EXIT_MOV_CRX:
11958	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11959	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
11960	SET_BOTH(CRX_READ);
11961	else
11962	SET_BOTH(CRX_WRITE);
11963	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
11964	break;
11965	case VMX_EXIT_MOV_DRX:
11966	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11967	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
11968	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
11969	SET_BOTH(DRX_READ);
11970	else
11971	SET_BOTH(DRX_WRITE);
11972	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
11973	break;
11974	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
11975	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
11976	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
11977	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
11978	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
11979	case VMX_EXIT_GDTR_IDTR_ACCESS:
11980	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11981	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
11982	{
11983	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
11984	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
11985	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
11986	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
11987	}
11988	break;
11989
11990	case VMX_EXIT_LDTR_TR_ACCESS:
11991	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11992	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
11993	{
11994	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
11995	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
11996	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
11997	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
11998	}
11999	break;
12000
12001	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
12002	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
12003	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
12004	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
12005	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
12006	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
12007	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
12008	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
12009	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
12010	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
12011	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
12012
12013	/* Events that aren't relevant at this point. */
12014	case VMX_EXIT_EXT_INT:
12015	case VMX_EXIT_INT_WINDOW:
12016	case VMX_EXIT_NMI_WINDOW:
12017	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12018	case VMX_EXIT_PREEMPT_TIMER:
12019	case VMX_EXIT_IO_INSTR:
12020	break;
12021
12022	/* Errors and unexpected events. */
12023	case VMX_EXIT_INIT_SIGNAL:
12024	case VMX_EXIT_SIPI:
12025	case VMX_EXIT_IO_SMI:
12026	case VMX_EXIT_SMI:
12027	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12028	case VMX_EXIT_ERR_MSR_LOAD:
12029	case VMX_EXIT_ERR_MACHINE_CHECK:
12030	case VMX_EXIT_PML_FULL:
12031	case VMX_EXIT_VIRTUALIZED_EOI:
12032	break;
12033
12034	default:
12035	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12036	break;
12037	}
12038	#undef SET_BOTH
12039	#undef SET_EXIT
12040
12041	/*
12042	* Dtrace tracepoints go first. We do them here at once so we don't
12043	* have to copy the guest state saving and stuff a few dozen times.
12044	* Down side is that we've got to repeat the switch, though this time
12045	* we use enmEvent since the probes are a subset of what DBGF does.
12046	*/
12047	if (fDtrace1 \|\| fDtrace2)
12048	{
12049	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12050	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12051	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12052	switch (enmEvent1)
12053	{
12054	/** @todo consider which extra parameters would be helpful for each probe. */
12055	case DBGFEVENT_END: break;
12056	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
12057	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
12058	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
12059	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
12060	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
12061	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
12062	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
12063	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
12064	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
12065	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
12066	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
12067	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
12068	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
12069	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
12070	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
12071	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
12072	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
12073	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
12074	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12075	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
12076	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
12077	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
12078	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
12079	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
12080	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
12081	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
12082	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
12083	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12084	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12085	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12086	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12087	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
12088	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
12089	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
12090	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
12091	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
12092	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
12093	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
12094	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
12095	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
12096	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
12097	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
12098	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
12099	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
12100	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
12101	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
12102	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
12103	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
12104	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
12105	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
12106	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
12107	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
12108	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
12109	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
12110	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
12111	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
12112	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
12113	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
12114	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
12115	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
12116	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
12117	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
12118	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
12119	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
12120	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
12121	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
12122	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
12123	}
12124	switch (enmEvent2)
12125	{
12126	/** @todo consider which extra parameters would be helpful for each probe. */
12127	case DBGFEVENT_END: break;
12128	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
12129	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
12130	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
12131	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
12132	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
12133	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
12134	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
12135	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
12136	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
12137	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12138	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12139	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
12140	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
12141	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
12142	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
12143	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
12144	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
12145	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
12146	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
12147	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
12148	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
12149	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
12150	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
12151	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
12152	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
12153	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
12154	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
12155	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
12156	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
12157	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
12158	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
12159	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
12160	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
12161	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
12162	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
12163	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
12164	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
12165	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
12166	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
12167	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
12168	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
12169	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
12170	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
12171	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
12172	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
12173	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
12174	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
12175	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
12176	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
12177	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
12178	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
12179	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
12180	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
12181	}
12182	}
12183
12184	/*
12185	* Fire of the DBGF event, if enabled (our check here is just a quick one,
12186	* the DBGF call will do a full check).
12187	*
12188	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
12189	* Note! If we have to events, we prioritize the first, i.e. the instruction
12190	* one, in order to avoid event nesting.
12191	*/
12192	PVM pVM = pVCpu->CTX_SUFF(pVM);
12193	if ( enmEvent1 != DBGFEVENT_END
12194	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
12195	{
12196	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12197	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
12198	if (rcStrict != VINF_SUCCESS)
12199	return rcStrict;
12200	}
12201	else if ( enmEvent2 != DBGFEVENT_END
12202	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
12203	{
12204	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12205	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
12206	if (rcStrict != VINF_SUCCESS)
12207	return rcStrict;
12208	}
12209
12210	return VINF_SUCCESS;
12211	}
12212
12213
12214	/**
12215	* Single-stepping VM-exit filtering.
12216	*
12217	* This is preprocessing the VM-exits and deciding whether we've gotten far
12218	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
12219	* handling is performed.
12220	*
12221	* @returns Strict VBox status code (i.e. informational status codes too).
12222	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
12223	* @param pVmxTransient The VMX-transient structure.
12224	* @param pDbgState The debug state.
12225	*/
12226	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
12227	{
12228	/*
12229	* Expensive (saves context) generic dtrace VM-exit probe.
12230	*/
12231	uint32_t const uExitReason = pVmxTransient->uExitReason;
12232	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
12233	{ /* more likely */ }
12234	else
12235	{
12236	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12237	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
12238	AssertRC(rc);
12239	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
12240	}
12241
12242	/*
12243	* Check for host NMI, just to get that out of the way.
12244	*/
12245	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
12246	{ /* normally likely */ }
12247	else
12248	{
12249	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12250	AssertRCReturn(rc2, rc2);
12251	uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12252	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
12253	return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
12254	}
12255
12256	/*
12257	* Check for single stepping event if we're stepping.
12258	*/
12259	if (pVCpu->hm.s.fSingleInstruction)
12260	{
12261	switch (uExitReason)
12262	{
12263	case VMX_EXIT_MTF:
12264	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
12265
12266	/* Various events: */
12267	case VMX_EXIT_XCPT_OR_NMI:
12268	case VMX_EXIT_EXT_INT:
12269	case VMX_EXIT_TRIPLE_FAULT:
12270	case VMX_EXIT_INT_WINDOW:
12271	case VMX_EXIT_NMI_WINDOW:
12272	case VMX_EXIT_TASK_SWITCH:
12273	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12274	case VMX_EXIT_APIC_ACCESS:
12275	case VMX_EXIT_EPT_VIOLATION:
12276	case VMX_EXIT_EPT_MISCONFIG:
12277	case VMX_EXIT_PREEMPT_TIMER:
12278
12279	/* Instruction specific VM-exits: */
12280	case VMX_EXIT_CPUID:
12281	case VMX_EXIT_GETSEC:
12282	case VMX_EXIT_HLT:
12283	case VMX_EXIT_INVD:
12284	case VMX_EXIT_INVLPG:
12285	case VMX_EXIT_RDPMC:
12286	case VMX_EXIT_RDTSC:
12287	case VMX_EXIT_RSM:
12288	case VMX_EXIT_VMCALL:
12289	case VMX_EXIT_VMCLEAR:
12290	case VMX_EXIT_VMLAUNCH:
12291	case VMX_EXIT_VMPTRLD:
12292	case VMX_EXIT_VMPTRST:
12293	case VMX_EXIT_VMREAD:
12294	case VMX_EXIT_VMRESUME:
12295	case VMX_EXIT_VMWRITE:
12296	case VMX_EXIT_VMXOFF:
12297	case VMX_EXIT_VMXON:
12298	case VMX_EXIT_MOV_CRX:
12299	case VMX_EXIT_MOV_DRX:
12300	case VMX_EXIT_IO_INSTR:
12301	case VMX_EXIT_RDMSR:
12302	case VMX_EXIT_WRMSR:
12303	case VMX_EXIT_MWAIT:
12304	case VMX_EXIT_MONITOR:
12305	case VMX_EXIT_PAUSE:
12306	case VMX_EXIT_GDTR_IDTR_ACCESS:
12307	case VMX_EXIT_LDTR_TR_ACCESS:
12308	case VMX_EXIT_INVEPT:
12309	case VMX_EXIT_RDTSCP:
12310	case VMX_EXIT_INVVPID:
12311	case VMX_EXIT_WBINVD:
12312	case VMX_EXIT_XSETBV:
12313	case VMX_EXIT_RDRAND:
12314	case VMX_EXIT_INVPCID:
12315	case VMX_EXIT_VMFUNC:
12316	case VMX_EXIT_RDSEED:
12317	case VMX_EXIT_XSAVES:
12318	case VMX_EXIT_XRSTORS:
12319	{
12320	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12321	AssertRCReturn(rc, rc);
12322	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
12323	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
12324	return VINF_EM_DBG_STEPPED;
12325	break;
12326	}
12327
12328	/* Errors and unexpected events: */
12329	case VMX_EXIT_INIT_SIGNAL:
12330	case VMX_EXIT_SIPI:
12331	case VMX_EXIT_IO_SMI:
12332	case VMX_EXIT_SMI:
12333	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12334	case VMX_EXIT_ERR_MSR_LOAD:
12335	case VMX_EXIT_ERR_MACHINE_CHECK:
12336	case VMX_EXIT_PML_FULL:
12337	case VMX_EXIT_VIRTUALIZED_EOI:
12338	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
12339	break;
12340
12341	default:
12342	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
12343	break;
12344	}
12345	}
12346
12347	/*
12348	* Check for debugger event breakpoints and dtrace probes.
12349	*/
12350	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
12351	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
12352	{
12353	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
12354	if (rcStrict != VINF_SUCCESS)
12355	return rcStrict;
12356	}
12357
12358	/*
12359	* Normal processing.
12360	*/
12361	#ifdef HMVMX_USE_FUNCTION_TABLE
12362	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
12363	#else
12364	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
12365	#endif
12366	}
12367
12368
12369	/**
12370	* Single steps guest code using hardware-assisted VMX.
12371	*
12372	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
12373	* but single-stepping through the hypervisor debugger.
12374	*
12375	* @returns Strict VBox status code (i.e. informational status codes too).
12376	* @param pVCpu The cross context virtual CPU structure.
12377	* @param pcLoops Pointer to the number of executed loops.
12378	*
12379	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
12380	*/
12381	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, uint32_t *pcLoops)
12382	{
12383	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
12384	Assert(pcLoops);
12385	Assert(*pcLoops <= cMaxResumeLoops);
12386
12387	VMXTRANSIENT VmxTransient;
12388	RT_ZERO(VmxTransient);
12389	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
12390
12391	/* Set HMCPU indicators. */
12392	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
12393	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
12394	pVCpu->hm.s.fDebugWantRdTscExit = false;
12395	pVCpu->hm.s.fUsingDebugLoop = true;
12396
12397	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
12398	VMXRUNDBGSTATE DbgState;
12399	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
12400	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12401
12402	/*
12403	* The loop.
12404	*/
12405	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
12406	for (;;)
12407	{
12408	Assert(!HMR0SuspendPending());
12409	HMVMX_ASSERT_CPU_SAFE(pVCpu);
12410	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
12411	bool fStepping = pVCpu->hm.s.fSingleInstruction;
12412
12413	/* Set up VM-execution controls the next two can respond to. */
12414	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12415
12416	/*
12417	* Preparatory work for running guest code, this may force us to
12418	* return to ring-3.
12419	*
12420	* Warning! This bugger disables interrupts on VINF_SUCCESS!
12421	*/
12422	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
12423	if (rcStrict != VINF_SUCCESS)
12424	break;
12425
12426	/* Interrupts are disabled at this point! */
12427	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
12428
12429	/* Override any obnoxious code in the above two calls. */
12430	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12431
12432	/*
12433	* Finally execute the guest.
12434	*/
12435	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
12436
12437	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
12438	/* Interrupts are re-enabled at this point! */
12439
12440	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
12441	if (RT_SUCCESS(rcRun))
12442	{ /* very likely */ }
12443	else
12444	{
12445	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12446	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12447	return rcRun;
12448	}
12449
12450	/* Profile the VM-exit. */
12451	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12452	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12453	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12454	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12455	HMVMX_START_EXIT_DISPATCH_PROF();
12456
12457	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12458
12459	/*
12460	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12461	*/
12462	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12463	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12464	if (rcStrict != VINF_SUCCESS)
12465	break;
12466	if (++(*pcLoops) > cMaxResumeLoops)
12467	{
12468	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12469	rcStrict = VINF_EM_RAW_INTERRUPT;
12470	break;
12471	}
12472
12473	/*
12474	* Stepping: Did the RIP change, if so, consider it a single step.
12475	* Otherwise, make sure one of the TFs gets set.
12476	*/
12477	if (fStepping)
12478	{
12479	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12480	AssertRC(rc);
12481	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12482	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12483	{
12484	rcStrict = VINF_EM_DBG_STEPPED;
12485	break;
12486	}
12487	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12488	}
12489
12490	/*
12491	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12492	*/
12493	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12494	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12495	}
12496
12497	/*
12498	* Clear the X86_EFL_TF if necessary.
12499	*/
12500	if (pVCpu->hm.s.fClearTrapFlag)
12501	{
12502	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12503	AssertRC(rc);
12504	pVCpu->hm.s.fClearTrapFlag = false;
12505	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12506	}
12507	/** @todo there seems to be issues with the resume flag when the monitor trap
12508	* flag is pending without being used. Seen early in bios init when
12509	* accessing APIC page in protected mode. */
12510
12511	/*
12512	* Restore VM-exit control settings as we may not re-enter this function the
12513	* next time around.
12514	*/
12515	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12516
12517	/* Restore HMCPU indicators. */
12518	pVCpu->hm.s.fUsingDebugLoop = false;
12519	pVCpu->hm.s.fDebugWantRdTscExit = false;
12520	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
12521
12522	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
12523	return rcStrict;
12524	}
12525
12526
12527	/** @} */
12528
12529
12530	/**
12531	* Checks if any expensive dtrace probes are enabled and we should go to the
12532	* debug loop.
12533	*
12534	* @returns true if we should use debug loop, false if not.
12535	*/
12536	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
12537	{
12538	/* It's probably faster to OR the raw 32-bit counter variables together.
12539	Since the variables are in an array and the probes are next to one
12540	another (more or less), we have good locality. So, better read
12541	eight-nine cache lines ever time and only have one conditional, than
12542	128+ conditionals, right? */
12543	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
12544	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
12545	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
12546	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
12547	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
12548	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
12549	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
12550	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
12551	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
12552	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
12553	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
12554	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
12555	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
12556	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
12557	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
12558	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
12559	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
12560	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
12561	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
12562	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
12563	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
12564	) != 0
12565	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
12566	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
12567	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
12568	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
12569	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
12570	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
12571	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
12572	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
12573	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
12574	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
12575	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
12576	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
12577	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
12578	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
12579	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
12580	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
12581	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
12582	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
12583	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
12584	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
12585	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
12586	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
12587	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
12588	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
12589	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
12590	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
12591	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
12592	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
12593	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
12594	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
12595	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
12596	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
12597	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
12598	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
12599	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
12600	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
12601	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
12602	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
12603	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
12604	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
12605	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
12606	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
12607	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
12608	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
12609	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
12610	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
12611	) != 0
12612	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
12613	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
12614	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
12615	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
12616	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
12617	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
12618	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
12619	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
12620	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
12621	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
12622	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
12623	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
12624	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
12625	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
12626	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
12627	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
12628	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
12629	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
12630	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
12631	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
12632	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
12633	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
12634	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
12635	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
12636	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
12637	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
12638	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
12639	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
12640	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
12641	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
12642	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
12643	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
12644	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
12645	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
12646	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
12647	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
12648	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
12649	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
12650	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
12651	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
12652	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
12653	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
12654	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
12655	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
12656	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
12657	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
12658	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
12659	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
12660	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
12661	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
12662	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
12663	) != 0;
12664	}
12665
12666
12667	/**
12668	* Runs the guest using hardware-assisted VMX.
12669	*
12670	* @returns Strict VBox status code (i.e. informational status codes too).
12671	* @param pVCpu The cross context virtual CPU structure.
12672	*/
12673	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
12674	{
12675	AssertPtr(pVCpu);
12676	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12677	Assert(VMMRZCallRing3IsEnabled(pVCpu));
12678	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12679	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
12680
12681	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
12682
12683	VBOXSTRICTRC rcStrict;
12684	uint32_t cLoops = 0;
12685	for (;;)
12686	{
12687	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12688	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
12689	#else
12690	bool const fInNestedGuestMode = false;
12691	#endif
12692	if (!fInNestedGuestMode)
12693	{
12694	if ( !pVCpu->hm.s.fUseDebugLoop
12695	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
12696	&& !DBGFIsStepping(pVCpu)
12697	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
12698	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
12699	else
12700	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
12701	}
12702	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12703	else
12704	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
12705
12706	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
12707	{
12708	Assert(CPUMIsGuestInVmxNonRootMode(pCtx));
12709	continue;
12710	}
12711	if (rcStrict == VINF_VMX_VMEXIT)
12712	{
12713	Assert(!CPUMIsGuestInVmxNonRootMode(pCtx));
12714	continue;
12715	}
12716	#endif
12717	break;
12718	}
12719
12720	int const rcLoop = VBOXSTRICTRC_VAL(rcStrict);
12721	switch (rcLoop)
12722	{
12723	case VERR_EM_INTERPRETER: rcStrict = VINF_EM_RAW_EMULATE_INSTR; break;
12724	case VINF_EM_RESET: rcStrict = VINF_EM_TRIPLE_FAULT; break;
12725	}
12726
12727	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
12728	if (RT_FAILURE(rc2))
12729	{
12730	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
12731	rcStrict = rc2;
12732	}
12733	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12734	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
12735	return rcStrict;
12736	}
12737
12738
12739	#ifndef HMVMX_USE_FUNCTION_TABLE
12740	/**
12741	* Handles a guest VM-exit from hardware-assisted VMX execution.
12742	*
12743	* @returns Strict VBox status code (i.e. informational status codes too).
12744	* @param pVCpu The cross context virtual CPU structure.
12745	* @param pVmxTransient The VMX-transient structure.
12746	*/
12747	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12748	{
12749	#ifdef DEBUG_ramshankar
12750	#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
12751	do { \
12752	if (a_fSave != 0) \
12753	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL); \
12754	VBOXSTRICTRC rcStrict = a_CallExpr; \
12755	if (a_fSave != 0) \
12756	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
12757	return rcStrict; \
12758	} while (0)
12759	#else
12760	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
12761	#endif
12762	uint32_t const uExitReason = pVmxTransient->uExitReason;
12763	switch (uExitReason)
12764	{
12765	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
12766	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
12767	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
12768	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
12769	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
12770	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
12771	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
12772	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
12773	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
12774	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
12775	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
12776	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
12777	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
12778	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
12779	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
12780	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
12781	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
12782	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
12783	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
12784	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
12785	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
12786	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
12787	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
12788	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
12789	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
12790	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
12791	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
12792	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
12793	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
12794	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
12795	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12796	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
12797	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
12798	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
12799	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
12800	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
12801	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
12802	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
12803	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
12804	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
12805	case VMX_EXIT_INVVPID: VMEXIT_CALL_RET(0, hmR0VmxExitInvvpid(pVCpu, pVmxTransient));
12806	case VMX_EXIT_INVEPT: VMEXIT_CALL_RET(0, hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient));
12807	#else
12808	case VMX_EXIT_VMCLEAR:
12809	case VMX_EXIT_VMLAUNCH:
12810	case VMX_EXIT_VMPTRLD:
12811	case VMX_EXIT_VMPTRST:
12812	case VMX_EXIT_VMREAD:
12813	case VMX_EXIT_VMRESUME:
12814	case VMX_EXIT_VMWRITE:
12815	case VMX_EXIT_VMXOFF:
12816	case VMX_EXIT_VMXON:
12817	case VMX_EXIT_INVVPID:
12818	case VMX_EXIT_INVEPT:
12819	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12820	#endif
12821
12822	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
12823	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
12824	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
12825
12826	case VMX_EXIT_INIT_SIGNAL:
12827	case VMX_EXIT_SIPI:
12828	case VMX_EXIT_IO_SMI:
12829	case VMX_EXIT_SMI:
12830	case VMX_EXIT_ERR_MSR_LOAD:
12831	case VMX_EXIT_ERR_MACHINE_CHECK:
12832	case VMX_EXIT_PML_FULL:
12833	case VMX_EXIT_VIRTUALIZED_EOI:
12834	case VMX_EXIT_GDTR_IDTR_ACCESS:
12835	case VMX_EXIT_LDTR_TR_ACCESS:
12836	case VMX_EXIT_APIC_WRITE:
12837	case VMX_EXIT_RDRAND:
12838	case VMX_EXIT_RSM:
12839	case VMX_EXIT_VMFUNC:
12840	case VMX_EXIT_ENCLS:
12841	case VMX_EXIT_RDSEED:
12842	case VMX_EXIT_XSAVES:
12843	case VMX_EXIT_XRSTORS:
12844	case VMX_EXIT_UMWAIT:
12845	case VMX_EXIT_TPAUSE:
12846	default:
12847	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12848	}
12849	#undef VMEXIT_CALL_RET
12850	}
12851	#endif /* !HMVMX_USE_FUNCTION_TABLE */
12852
12853
12854	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12855	/**
12856	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
12857	*
12858	* @returns Strict VBox status code (i.e. informational status codes too).
12859	* @param pVCpu The cross context virtual CPU structure.
12860	* @param pVmxTransient The VMX-transient structure.
12861	*/
12862	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12863	{
12864	uint32_t const uExitReason = pVmxTransient->uExitReason;
12865	switch (uExitReason)
12866	{
12867	case VMX_EXIT_EPT_MISCONFIG: return hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient);
12868	case VMX_EXIT_EPT_VIOLATION: return hmR0VmxExitEptViolation(pVCpu, pVmxTransient);
12869	case VMX_EXIT_XCPT_OR_NMI: return hmR0VmxExitXcptOrNmiNested(pVCpu, pVmxTransient);
12870	case VMX_EXIT_IO_INSTR: return hmR0VmxExitIoInstrNested(pVCpu, pVmxTransient);
12871	case VMX_EXIT_HLT: return hmR0VmxExitHltNested(pVCpu, pVmxTransient);
12872
12873	/*
12874	* We shouldn't direct host physical interrupts to the nested-guest.
12875	*/
12876	case VMX_EXIT_EXT_INT:
12877	return hmR0VmxExitExtInt(pVCpu, pVmxTransient);
12878
12879	/*
12880	* Instructions that cause VM-exits unconditionally or the condition is
12881	* always is taken solely from the guest hypervisor (meaning if the VM-exit
12882	* happens, it's guaranteed to be a nested-guest VM-exit).
12883	*
12884	* - Provides VM-exit instruction length ONLY.
12885	*/
12886	case VMX_EXIT_CPUID: /* Unconditional. */
12887	case VMX_EXIT_VMCALL:
12888	case VMX_EXIT_GETSEC:
12889	case VMX_EXIT_INVD:
12890	case VMX_EXIT_XSETBV:
12891	case VMX_EXIT_VMLAUNCH:
12892	case VMX_EXIT_VMRESUME:
12893	case VMX_EXIT_VMXOFF:
12894	case VMX_EXIT_ENCLS: /* Condition specified solely by guest hypervisor. */
12895	case VMX_EXIT_VMFUNC:
12896	return hmR0VmxExitInstrNested(pVCpu, pVmxTransient);
12897
12898	/*
12899	* Instructions that cause VM-exits unconditionally or the condition is
12900	* always is taken solely from the guest hypervisor (meaning if the VM-exit
12901	* happens, it's guaranteed to be a nested-guest VM-exit).
12902	*
12903	* - Provides VM-exit instruction length.
12904	* - Provides VM-exit information.
12905	* - Optionally provides Exit qualification.
12906	*
12907	* Since Exit qualification is 0 for all VM-exits where it is not
12908	* applicable, reading and passing it to the guest should produce
12909	* defined behavior.
12910	*
12911	* See Intel spec. 27.2.1 "Basic VM-Exit Information".
12912	*/
12913	case VMX_EXIT_INVEPT: /* Unconditional. */
12914	case VMX_EXIT_INVVPID:
12915	case VMX_EXIT_VMCLEAR:
12916	case VMX_EXIT_VMPTRLD:
12917	case VMX_EXIT_VMPTRST:
12918	case VMX_EXIT_VMXON:
12919	case VMX_EXIT_GDTR_IDTR_ACCESS: /* Condition specified solely by guest hypervisor. */
12920	case VMX_EXIT_LDTR_TR_ACCESS:
12921	case VMX_EXIT_RDRAND:
12922	case VMX_EXIT_RDSEED:
12923	case VMX_EXIT_XSAVES:
12924	case VMX_EXIT_XRSTORS:
12925	case VMX_EXIT_UMWAIT:
12926	case VMX_EXIT_TPAUSE:
12927	return hmR0VmxExitInstrWithInfoNested(pVCpu, pVmxTransient);
12928
12929	case VMX_EXIT_RDTSC: return hmR0VmxExitRdtscNested(pVCpu, pVmxTransient);
12930	case VMX_EXIT_RDTSCP: return hmR0VmxExitRdtscpNested(pVCpu, pVmxTransient);
12931	case VMX_EXIT_RDMSR: return hmR0VmxExitRdmsrNested(pVCpu, pVmxTransient);
12932	case VMX_EXIT_WRMSR: return hmR0VmxExitWrmsrNested(pVCpu, pVmxTransient);
12933	case VMX_EXIT_INVLPG: return hmR0VmxExitInvlpgNested(pVCpu, pVmxTransient);
12934	case VMX_EXIT_INVPCID: return hmR0VmxExitInvpcidNested(pVCpu, pVmxTransient);
12935	case VMX_EXIT_TASK_SWITCH: return hmR0VmxExitTaskSwitchNested(pVCpu, pVmxTransient);
12936	case VMX_EXIT_WBINVD: return hmR0VmxExitWbinvdNested(pVCpu, pVmxTransient);
12937	case VMX_EXIT_MTF: return hmR0VmxExitMtfNested(pVCpu, pVmxTransient);
12938	case VMX_EXIT_APIC_ACCESS: return hmR0VmxExitApicAccessNested(pVCpu, pVmxTransient);
12939	case VMX_EXIT_APIC_WRITE: return hmR0VmxExitApicWriteNested(pVCpu, pVmxTransient);
12940	case VMX_EXIT_VIRTUALIZED_EOI: return hmR0VmxExitVirtEoiNested(pVCpu, pVmxTransient);
12941	case VMX_EXIT_MOV_CRX: return hmR0VmxExitMovCRxNested(pVCpu, pVmxTransient);
12942	case VMX_EXIT_INT_WINDOW: return hmR0VmxExitIntWindowNested(pVCpu, pVmxTransient);
12943	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindowNested(pVCpu, pVmxTransient);
12944	case VMX_EXIT_TPR_BELOW_THRESHOLD: return hmR0VmxExitTprBelowThresholdNested(pVCpu, pVmxTransient);
12945	case VMX_EXIT_MWAIT: return hmR0VmxExitMwaitNested(pVCpu, pVmxTransient);
12946	case VMX_EXIT_MONITOR: return hmR0VmxExitMonitorNested(pVCpu, pVmxTransient);
12947	case VMX_EXIT_PAUSE: return hmR0VmxExitPauseNested(pVCpu, pVmxTransient);
12948
12949	case VMX_EXIT_PREEMPT_TIMER:
12950	{
12951	/** @todo NSTVMX: Preempt timer. */
12952	return hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient);
12953	}
12954
12955	case VMX_EXIT_MOV_DRX: return hmR0VmxExitMovDRxNested(pVCpu, pVmxTransient);
12956	case VMX_EXIT_RDPMC: return hmR0VmxExitRdpmcNested(pVCpu, pVmxTransient);
12957
12958	case VMX_EXIT_VMREAD:
12959	case VMX_EXIT_VMWRITE: return hmR0VmxExitVmreadVmwriteNested(pVCpu, pVmxTransient);
12960
12961	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFaultNested(pVCpu, pVmxTransient);
12962	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestStateNested(pVCpu, pVmxTransient);
12963
12964	case VMX_EXIT_INIT_SIGNAL:
12965	case VMX_EXIT_SIPI:
12966	case VMX_EXIT_IO_SMI:
12967	case VMX_EXIT_SMI:
12968	case VMX_EXIT_ERR_MSR_LOAD:
12969	case VMX_EXIT_ERR_MACHINE_CHECK:
12970	case VMX_EXIT_PML_FULL:
12971	case VMX_EXIT_RSM:
12972	default:
12973	return hmR0VmxExitErrUnexpected(pVCpu, pVmxTransient);
12974	}
12975	}
12976	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
12977
12978
12979	/** @name VM-exit helpers.
12980	* @{
12981	*/
12982	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12983	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit helpers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
12984	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12985
12986	/** Macro for VM-exits called unexpectedly. */
12987	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_HmError) \
12988	do { \
12989	(a_pVCpu)->hm.s.u32HMError = (a_HmError); \
12990	return VERR_VMX_UNEXPECTED_EXIT; \
12991	} while (0)
12992
12993	#ifdef VBOX_STRICT
12994	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
12995	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
12996	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
12997
12998	# define HMVMX_ASSERT_PREEMPT_CPUID() \
12999	do { \
13000	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
13001	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
13002	} while (0)
13003
13004	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13005	do { \
13006	AssertPtr((a_pVCpu)); \
13007	AssertPtr((a_pVmxTransient)); \
13008	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
13009	Assert((a_pVmxTransient)->pVmcsInfo); \
13010	Assert(ASMIntAreEnabled()); \
13011	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
13012	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
13013	Log4Func(("vcpu[%RU32]\n", (a_pVCpu)->idCpu)); \
13014	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
13015	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
13016	HMVMX_ASSERT_PREEMPT_CPUID(); \
13017	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
13018	} while (0)
13019
13020	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13021	do { \
13022	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); \
13023	Assert((a_pVmxTransient)->fIsNestedGuest); \
13024	} while (0)
13025
13026	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13027	do { \
13028	Log4Func(("\n")); \
13029	} while (0)
13030	#else
13031	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13032	do { \
13033	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
13034	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
13035	} while (0)
13036
13037	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
13038	do { HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); } while (0)
13039
13040	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
13041	#endif
13042
13043	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13044	/** Macro that does the necessary privilege checks and intercepted VM-exits for
13045	* guests that attempted to execute a VMX instruction. */
13046	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
13047	do \
13048	{ \
13049	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
13050	if (rcStrictTmp == VINF_SUCCESS) \
13051	{ /* likely */ } \
13052	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
13053	{ \
13054	Assert((a_pVCpu)->hm.s.Event.fPending); \
13055	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
13056	return VINF_SUCCESS; \
13057	} \
13058	else \
13059	{ \
13060	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
13061	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
13062	} \
13063	} while (0)
13064
13065	/** Macro that decodes a memory operand for an VM-exit caused by an instruction. */
13066	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
13067	do \
13068	{ \
13069	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
13070	(a_pGCPtrEffAddr)); \
13071	if (rcStrictTmp == VINF_SUCCESS) \
13072	{ /* likely */ } \
13073	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
13074	{ \
13075	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
13076	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
13077	NOREF(uXcptTmp); \
13078	return VINF_SUCCESS; \
13079	} \
13080	else \
13081	{ \
13082	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
13083	return rcStrictTmp; \
13084	} \
13085	} while (0)
13086	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13087
13088
13089	/**
13090	* Advances the guest RIP by the specified number of bytes.
13091	*
13092	* @param pVCpu The cross context virtual CPU structure.
13093	* @param cbInstr Number of bytes to advance the RIP by.
13094	*
13095	* @remarks No-long-jump zone!!!
13096	*/
13097	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
13098	{
13099	/* Advance the RIP. */
13100	pVCpu->cpum.GstCtx.rip += cbInstr;
13101	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
13102
13103	/* Update interrupt inhibition. */
13104	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
13105	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
13106	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
13107	}
13108
13109
13110	/**
13111	* Advances the guest RIP after reading it from the VMCS.
13112	*
13113	* @returns VBox status code, no informational status codes.
13114	* @param pVCpu The cross context virtual CPU structure.
13115	* @param pVmxTransient The VMX-transient structure.
13116	*
13117	* @remarks No-long-jump zone!!!
13118	*/
13119	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13120	{
13121	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13122	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
13123	AssertRCReturn(rc, rc);
13124
13125	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
13126	return VINF_SUCCESS;
13127	}
13128
13129
13130	/**
13131	* Handle a condition that occurred while delivering an event through the guest or
13132	* nested-guest IDT.
13133	*
13134	* @returns Strict VBox status code (i.e. informational status codes too).
13135	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
13136	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
13137	* to continue execution of the guest which will delivery the \#DF.
13138	* @retval VINF_EM_RESET if we detected a triple-fault condition.
13139	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
13140	*
13141	* @param pVCpu The cross context virtual CPU structure.
13142	* @param pVmxTransient The VMX-transient structure.
13143	*
13144	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13145	* Additionally, HMVMX_READ_EXIT_QUALIFICATION is required if the VM-exit
13146	* is due to an EPT violation, PML full or SPP-related event.
13147	*
13148	* @remarks No-long-jump zone!!!
13149	*/
13150	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13151	{
13152	Assert(!pVCpu->hm.s.Event.fPending);
13153	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
13154	if ( pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
13155	\|\| pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
13156	\|\| pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
13157	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_EXIT_QUALIFICATION);
13158
13159	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
13160	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13161	uint32_t const uIdtVectorInfo = pVmxTransient->uIdtVectoringInfo;
13162	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
13163	if (VMX_IDT_VECTORING_INFO_IS_VALID(uIdtVectorInfo))
13164	{
13165	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(uIdtVectorInfo);
13166	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(uIdtVectorInfo);
13167
13168	/*
13169	* If the event was a software interrupt (generated with INT n) or a software exception
13170	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
13171	* can handle the VM-exit and continue guest execution which will re-execute the
13172	* instruction rather than re-injecting the exception, as that can cause premature
13173	* trips to ring-3 before injection and involve TRPM which currently has no way of
13174	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
13175	* the problem).
13176	*/
13177	IEMXCPTRAISE enmRaise;
13178	IEMXCPTRAISEINFO fRaiseInfo;
13179	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
13180	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
13181	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
13182	{
13183	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
13184	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13185	}
13186	else if (VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo))
13187	{
13188	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
13189	uint8_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
13190	Assert(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT);
13191
13192	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
13193	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
13194
13195	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
13196
13197	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
13198	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
13199	{
13200	pVmxTransient->fVectoringPF = true;
13201	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13202	}
13203	}
13204	else
13205	{
13206	/*
13207	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
13208	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
13209	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
13210	*/
13211	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
13212	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13213	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
13214	enmRaise = IEMXCPTRAISE_PREV_EVENT;
13215	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
13216	}
13217
13218	/*
13219	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
13220	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
13221	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
13222	* subsequent VM-entry would fail, see @bugref{7445}.
13223	*
13224	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception".
13225	*/
13226	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
13227	&& enmRaise == IEMXCPTRAISE_PREV_EVENT
13228	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
13229	&& CPUMIsGuestNmiBlocking(pVCpu))
13230	{
13231	CPUMSetGuestNmiBlocking(pVCpu, false);
13232	}
13233
13234	switch (enmRaise)
13235	{
13236	case IEMXCPTRAISE_CURRENT_XCPT:
13237	{
13238	Log4Func(("IDT: Pending secondary Xcpt: idtinfo=%#RX64 exitinfo=%#RX64\n", uIdtVectorInfo, uExitIntInfo));
13239	Assert(rcStrict == VINF_SUCCESS);
13240	break;
13241	}
13242
13243	case IEMXCPTRAISE_PREV_EVENT:
13244	{
13245	uint32_t u32ErrCode;
13246	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(uIdtVectorInfo))
13247	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
13248	else
13249	u32ErrCode = 0;
13250
13251	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
13252	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflect);
13253	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(uIdtVectorInfo), 0 /* cbInstr */,
13254	u32ErrCode, pVCpu->cpum.GstCtx.cr2);
13255
13256	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
13257	pVCpu->hm.s.Event.u32ErrCode));
13258	Assert(rcStrict == VINF_SUCCESS);
13259	break;
13260	}
13261
13262	case IEMXCPTRAISE_REEXEC_INSTR:
13263	Assert(rcStrict == VINF_SUCCESS);
13264	break;
13265
13266	case IEMXCPTRAISE_DOUBLE_FAULT:
13267	{
13268	/*
13269	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
13270	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
13271	*/
13272	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
13273	{
13274	pVmxTransient->fVectoringDoublePF = true;
13275	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
13276	pVCpu->cpum.GstCtx.cr2));
13277	rcStrict = VINF_SUCCESS;
13278	}
13279	else
13280	{
13281	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectConvertDF);
13282	hmR0VmxSetPendingXcptDF(pVCpu);
13283	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
13284	uIdtVector, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
13285	rcStrict = VINF_HM_DOUBLE_FAULT;
13286	}
13287	break;
13288	}
13289
13290	case IEMXCPTRAISE_TRIPLE_FAULT:
13291	{
13292	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector,
13293	VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
13294	rcStrict = VINF_EM_RESET;
13295	break;
13296	}
13297
13298	case IEMXCPTRAISE_CPU_HANG:
13299	{
13300	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
13301	rcStrict = VERR_EM_GUEST_CPU_HANG;
13302	break;
13303	}
13304
13305	default:
13306	{
13307	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
13308	rcStrict = VERR_VMX_IPE_2;
13309	break;
13310	}
13311	}
13312	}
13313	else if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
13314	&& !CPUMIsGuestNmiBlocking(pVCpu))
13315	{
13316	if ( VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo)
13317	&& VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo) != X86_XCPT_DF
13318	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(uExitIntInfo))
13319	{
13320	/*
13321	* Execution of IRET caused a fault when NMI blocking was in effect (i.e we're in
13322	* the guest or nested-guest NMI handler). We need to set the block-by-NMI field so
13323	* that NMIs remain blocked until the IRET execution is completed.
13324	*
13325	* See Intel spec. 31.7.1.2 "Resuming Guest Software After Handling An Exception".
13326	*/
13327	CPUMSetGuestNmiBlocking(pVCpu, true);
13328	Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
13329	}
13330	else if ( pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
13331	\|\| pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
13332	\|\| pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
13333	{
13334	/*
13335	* Execution of IRET caused an EPT violation, page-modification log-full event or
13336	* SPP-related event VM-exit when NMI blocking was in effect (i.e. we're in the
13337	* guest or nested-guest NMI handler). We need to set the block-by-NMI field so
13338	* that NMIs remain blocked until the IRET execution is completed.
13339	*
13340	* See Intel spec. 27.2.3 "Information about NMI unblocking due to IRET"
13341	*/
13342	if (VMX_EXIT_QUAL_EPT_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitQual))
13343	{
13344	CPUMSetGuestNmiBlocking(pVCpu, true);
13345	Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
13346	}
13347	}
13348	}
13349
13350	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
13351	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
13352	return rcStrict;
13353	}
13354
13355
13356	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13357	/**
13358	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
13359	* guest attempting to execute a VMX instruction.
13360	*
13361	* @returns Strict VBox status code (i.e. informational status codes too).
13362	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
13363	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
13364	*
13365	* @param pVCpu The cross context virtual CPU structure.
13366	* @param uExitReason The VM-exit reason.
13367	*
13368	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
13369	* @remarks No-long-jump zone!!!
13370	*/
13371	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
13372	{
13373	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
13374	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13375
13376	/*
13377	* The physical CPU would have already checked the CPU mode/code segment.
13378	* We shall just assert here for paranoia.
13379	* See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
13380	*/
13381	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
13382	Assert( !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
13383	\|\| CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));
13384
13385	if (uExitReason == VMX_EXIT_VMXON)
13386	{
13387	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
13388
13389	/*
13390	* We check CR4.VMXE because it is required to be always set while in VMX operation
13391	* by physical CPUs and our CR4 read-shadow is only consulted when executing specific
13392	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
13393	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
13394	*/
13395	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
13396	{
13397	Log4Func(("CR4.VMXE is not set -> #UD\n"));
13398	hmR0VmxSetPendingXcptUD(pVCpu);
13399	return VINF_HM_PENDING_XCPT;
13400	}
13401	}
13402	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
13403	{
13404	/*
13405	* The guest has not entered VMX operation but attempted to execute a VMX instruction
13406	* (other than VMXON), we need to raise a #UD.
13407	*/
13408	Log4Func(("Not in VMX root mode -> #UD\n"));
13409	hmR0VmxSetPendingXcptUD(pVCpu);
13410	return VINF_HM_PENDING_XCPT;
13411	}
13412
13413	/* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
13414	return VINF_SUCCESS;
13415	}
13416
13417	/**
13418	* Decodes the memory operand of an instruction that caused a VM-exit.
13419	*
13420	* The Exit qualification field provides the displacement field for memory
13421	* operand instructions, if any.
13422	*
13423	* @returns Strict VBox status code (i.e. informational status codes too).
13424	* @retval VINF_SUCCESS if the operand was successfully decoded.
13425	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
13426	* operand.
13427	* @param pVCpu The cross context virtual CPU structure.
13428	* @param uExitInstrInfo The VM-exit instruction information field.
13429	* @param enmMemAccess The memory operand's access type (read or write).
13430	* @param GCPtrDisp The instruction displacement field, if any. For
13431	* RIP-relative addressing pass RIP + displacement here.
13432	* @param pGCPtrMem Where to store the effective destination memory address.
13433	*
13434	* @remarks Warning! This function ASSUMES the instruction cannot be used in real or
13435	* virtual-8086 mode hence skips those checks while verifying if the
13436	* segment is valid.
13437	*/
13438	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
13439	PRTGCPTR pGCPtrMem)
13440	{
13441	Assert(pGCPtrMem);
13442	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
13443	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
13444	\| CPUMCTX_EXTRN_CR0);
13445
13446	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
13447	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
13448	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
13449
13450	VMXEXITINSTRINFO ExitInstrInfo;
13451	ExitInstrInfo.u = uExitInstrInfo;
13452	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
13453	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
13454	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
13455	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
13456	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
13457	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
13458	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
13459	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
13460	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
13461
13462	/*
13463	* Validate instruction information.
13464	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
13465	*/
13466	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
13467	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
13468	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
13469	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
13470	AssertLogRelMsgReturn(fIsMemOperand,
13471	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
13472
13473	/*
13474	* Compute the complete effective address.
13475	*
13476	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
13477	* See AMD spec. 4.5.2 "Segment Registers".
13478	*/
13479	RTGCPTR GCPtrMem = GCPtrDisp;
13480	if (fBaseRegValid)
13481	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
13482	if (fIdxRegValid)
13483	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
13484
13485	RTGCPTR const GCPtrOff = GCPtrMem;
13486	if ( !fIsLongMode
13487	\|\| iSegReg >= X86_SREG_FS)
13488	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
13489	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
13490
13491	/*
13492	* Validate effective address.
13493	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
13494	*/
13495	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
13496	Assert(cbAccess > 0);
13497	if (fIsLongMode)
13498	{
13499	if (X86_IS_CANONICAL(GCPtrMem))
13500	{
13501	*pGCPtrMem = GCPtrMem;
13502	return VINF_SUCCESS;
13503	}
13504
13505	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
13506	* "Data Limit Checks in 64-bit Mode". */
13507	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
13508	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13509	return VINF_HM_PENDING_XCPT;
13510	}
13511
13512	/*
13513	* This is a watered down version of iemMemApplySegment().
13514	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
13515	* and segment CPL/DPL checks are skipped.
13516	*/
13517	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
13518	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
13519	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
13520
13521	/* Check if the segment is present and usable. */
13522	if ( pSel->Attr.n.u1Present
13523	&& !pSel->Attr.n.u1Unusable)
13524	{
13525	Assert(pSel->Attr.n.u1DescType);
13526	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
13527	{
13528	/* Check permissions for the data segment. */
13529	if ( enmMemAccess == VMXMEMACCESS_WRITE
13530	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
13531	{
13532	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
13533	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
13534	return VINF_HM_PENDING_XCPT;
13535	}
13536
13537	/* Check limits if it's a normal data segment. */
13538	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
13539	{
13540	if ( GCPtrFirst32 > pSel->u32Limit
13541	\|\| GCPtrLast32 > pSel->u32Limit)
13542	{
13543	Log4Func(("Data segment limit exceeded. "
13544	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
13545	GCPtrLast32, pSel->u32Limit));
13546	if (iSegReg == X86_SREG_SS)
13547	hmR0VmxSetPendingXcptSS(pVCpu, 0);
13548	else
13549	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13550	return VINF_HM_PENDING_XCPT;
13551	}
13552	}
13553	else
13554	{
13555	/* Check limits if it's an expand-down data segment.
13556	Note! The upper boundary is defined by the B bit, not the G bit! */
13557	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
13558	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
13559	{
13560	Log4Func(("Expand-down data segment limit exceeded. "
13561	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
13562	GCPtrLast32, pSel->u32Limit));
13563	if (iSegReg == X86_SREG_SS)
13564	hmR0VmxSetPendingXcptSS(pVCpu, 0);
13565	else
13566	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13567	return VINF_HM_PENDING_XCPT;
13568	}
13569	}
13570	}
13571	else
13572	{
13573	/* Check permissions for the code segment. */
13574	if ( enmMemAccess == VMXMEMACCESS_WRITE
13575	\|\| ( enmMemAccess == VMXMEMACCESS_READ
13576	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
13577	{
13578	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
13579	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
13580	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13581	return VINF_HM_PENDING_XCPT;
13582	}
13583
13584	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
13585	if ( GCPtrFirst32 > pSel->u32Limit
13586	\|\| GCPtrLast32 > pSel->u32Limit)
13587	{
13588	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
13589	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
13590	if (iSegReg == X86_SREG_SS)
13591	hmR0VmxSetPendingXcptSS(pVCpu, 0);
13592	else
13593	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13594	return VINF_HM_PENDING_XCPT;
13595	}
13596	}
13597	}
13598	else
13599	{
13600	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
13601	hmR0VmxSetPendingXcptGP(pVCpu, 0);
13602	return VINF_HM_PENDING_XCPT;
13603	}
13604
13605	*pGCPtrMem = GCPtrMem;
13606	return VINF_SUCCESS;
13607	}
13608	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13609
13610
13611	/**
13612	* VM-exit helper for LMSW.
13613	*/
13614	static VBOXSTRICTRC hmR0VmxExitLmsw(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst)
13615	{
13616	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13617	AssertRCReturn(rc, rc);
13618
13619	VBOXSTRICTRC rcStrict = IEMExecDecodedLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst);
13620	AssertMsg( rcStrict == VINF_SUCCESS
13621	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13622
13623	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13624	if (rcStrict == VINF_IEM_RAISED_XCPT)
13625	{
13626	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13627	rcStrict = VINF_SUCCESS;
13628	}
13629
13630	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
13631	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13632	return rcStrict;
13633	}
13634
13635
13636	/**
13637	* VM-exit helper for CLTS.
13638	*/
13639	static VBOXSTRICTRC hmR0VmxExitClts(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr)
13640	{
13641	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13642	AssertRCReturn(rc, rc);
13643
13644	VBOXSTRICTRC rcStrict = IEMExecDecodedClts(pVCpu, cbInstr);
13645	AssertMsg( rcStrict == VINF_SUCCESS
13646	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13647
13648	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13649	if (rcStrict == VINF_IEM_RAISED_XCPT)
13650	{
13651	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13652	rcStrict = VINF_SUCCESS;
13653	}
13654
13655	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
13656	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13657	return rcStrict;
13658	}
13659
13660
13661	/**
13662	* VM-exit helper for MOV from CRx (CRx read).
13663	*/
13664	static VBOXSTRICTRC hmR0VmxExitMovFromCrX(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
13665	{
13666	Assert(iCrReg < 16);
13667	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
13668
13669	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13670	AssertRCReturn(rc, rc);
13671
13672	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
13673	AssertMsg( rcStrict == VINF_SUCCESS
13674	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13675
13676	if (iGReg == X86_GREG_xSP)
13677	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
13678	else
13679	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13680	#ifdef VBOX_WITH_STATISTICS
13681	switch (iCrReg)
13682	{
13683	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
13684	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
13685	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
13686	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
13687	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
13688	}
13689	#endif
13690	Log4Func(("CR%d Read access rcStrict=%Rrc\n", iCrReg, VBOXSTRICTRC_VAL(rcStrict)));
13691	return rcStrict;
13692	}
13693
13694
13695	/**
13696	* VM-exit helper for MOV to CRx (CRx write).
13697	*/
13698	static VBOXSTRICTRC hmR0VmxExitMovToCrX(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
13699	{
13700	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13701	AssertRCReturn(rc, rc);
13702
13703	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
13704	AssertMsg( rcStrict == VINF_SUCCESS
13705	\|\| rcStrict == VINF_IEM_RAISED_XCPT
13706	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13707
13708	switch (iCrReg)
13709	{
13710	case 0:
13711	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13712	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
13713	Log4Func(("CR0 write. rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
13714	break;
13715
13716	case 2:
13717	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
13718	/* Nothing to do here, CR2 it's not part of the VMCS. */
13719	break;
13720
13721	case 3:
13722	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
13723	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
13724	Log4Func(("CR3 write. rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
13725	break;
13726
13727	case 4:
13728	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
13729	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
13730	Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
13731	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
13732	break;
13733
13734	case 8:
13735	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13736	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
13737	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
13738	break;
13739
13740	default:
13741	AssertMsgFailed(("Invalid CRx register %#x\n", iCrReg));
13742	break;
13743	}
13744
13745	if (rcStrict == VINF_IEM_RAISED_XCPT)
13746	{
13747	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13748	rcStrict = VINF_SUCCESS;
13749	}
13750	return rcStrict;
13751	}
13752
13753
13754	/**
13755	* VM-exit exception handler for \#PF (Page-fault exception).
13756	*
13757	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13758	*/
13759	static VBOXSTRICTRC hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13760	{
13761	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13762	PVM pVM = pVCpu->CTX_SUFF(pVM);
13763	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13764	AssertRCReturn(rc, rc);
13765
13766	if (!pVM->hm.s.fNestedPaging)
13767	{ /* likely */ }
13768	else
13769	{
13770	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13771	Assert(pVmxTransient->fIsNestedGuest \|\| pVCpu->hm.s.fUsingDebugLoop);
13772	#endif
13773	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13774	if (!pVmxTransient->fVectoringDoublePF)
13775	{
13776	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13777	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
13778	}
13779	else
13780	{
13781	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13782	Assert(!pVmxTransient->fIsNestedGuest);
13783	hmR0VmxSetPendingXcptDF(pVCpu);
13784	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13785	}
13786	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13787	return rc;
13788	}
13789
13790	Assert(!pVmxTransient->fIsNestedGuest);
13791
13792	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13793	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13794	if (pVmxTransient->fVectoringPF)
13795	{
13796	Assert(pVCpu->hm.s.Event.fPending);
13797	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13798	}
13799
13800	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13801	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13802	AssertRCReturn(rc, rc);
13803
13804	Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
13805	pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
13806
13807	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13808	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
13809
13810	Log4Func(("#PF: rc=%Rrc\n", rc));
13811	if (rc == VINF_SUCCESS)
13812	{
13813	/*
13814	* This is typically a shadow page table sync or a MMIO instruction. But we may have
13815	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13816	*/
13817	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13818	TRPMResetTrap(pVCpu);
13819	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13820	return rc;
13821	}
13822
13823	if (rc == VINF_EM_RAW_GUEST_TRAP)
13824	{
13825	if (!pVmxTransient->fVectoringDoublePF)
13826	{
13827	/* It's a guest page fault and needs to be reflected to the guest. */
13828	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13829	TRPMResetTrap(pVCpu);
13830	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13831	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13832	uGstErrorCode, pVmxTransient->uExitQual);
13833	}
13834	else
13835	{
13836	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13837	TRPMResetTrap(pVCpu);
13838	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13839	hmR0VmxSetPendingXcptDF(pVCpu);
13840	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13841	}
13842
13843	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13844	return VINF_SUCCESS;
13845	}
13846
13847	TRPMResetTrap(pVCpu);
13848	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13849	return rc;
13850	}
13851
13852
13853	/**
13854	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
13855	*
13856	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13857	*/
13858	static VBOXSTRICTRC hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13859	{
13860	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13861	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13862
13863	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
13864	AssertRCReturn(rc, rc);
13865
13866	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
13867	{
13868	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13869	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13870
13871	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13872	* provides VM-exit instruction length. If this causes problem later,
13873	* disassemble the instruction like it's done on AMD-V. */
13874	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13875	AssertRCReturn(rc2, rc2);
13876	return rc;
13877	}
13878
13879	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13880	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13881	return VINF_SUCCESS;
13882	}
13883
13884
13885	/**
13886	* VM-exit exception handler for \#BP (Breakpoint exception).
13887	*
13888	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13889	*/
13890	static VBOXSTRICTRC hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13891	{
13892	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13893	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13894
13895	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13896	AssertRCReturn(rc, rc);
13897
13898	if (!pVmxTransient->fIsNestedGuest)
13899	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx));
13900	else
13901	rc = VINF_EM_RAW_GUEST_TRAP;
13902	if (rc == VINF_EM_RAW_GUEST_TRAP)
13903	{
13904	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13905	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13906	}
13907
13908	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
13909	return rc;
13910	}
13911
13912
13913	/**
13914	* VM-exit exception handler for \#AC (Alignment-check exception).
13915	*
13916	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13917	*/
13918	static VBOXSTRICTRC hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13919	{
13920	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13921	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC);
13922
13923	/* Re-inject it. We'll detect any nesting before getting here. */
13924	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13925	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13926	return VINF_SUCCESS;
13927	}
13928
13929
13930	/**
13931	* VM-exit exception handler for \#DB (Debug exception).
13932	*
13933	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
13934	*/
13935	static VBOXSTRICTRC hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13936	{
13937	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13938	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13939
13940	/*
13941	* Get the DR6-like values from the Exit qualification and pass it to DBGF for processing.
13942	*/
13943	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13944	AssertRCReturn(rc, rc);
13945
13946	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13947	uint64_t const uDR6 = X86_DR6_INIT_VAL
13948	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
13949	\| X86_DR6_BD \| X86_DR6_BS));
13950
13951	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13952	if (!pVmxTransient->fIsNestedGuest)
13953	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13954	else
13955	rc = VINF_EM_RAW_GUEST_TRAP;
13956	Log6Func(("rc=%Rrc\n", rc));
13957	if (rc == VINF_EM_RAW_GUEST_TRAP)
13958	{
13959	/*
13960	* The exception was for the guest. Update DR6, DR7.GD and
13961	* IA32_DEBUGCTL.LBR before forwarding it.
13962	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
13963	*/
13964	VMMRZCallRing3Disable(pVCpu);
13965	HM_DISABLE_PREEMPT(pVCpu);
13966
13967	pCtx->dr[6] &= ~X86_DR6_B_MASK;
13968	pCtx->dr[6] \|= uDR6;
13969	if (CPUMIsGuestDebugStateActive(pVCpu))
13970	ASMSetDR6(pCtx->dr[6]);
13971
13972	HM_RESTORE_PREEMPT();
13973	VMMRZCallRing3Enable(pVCpu);
13974
13975	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
13976	AssertRCReturn(rc, rc);
13977
13978	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13979	pCtx->dr[7] &= ~X86_DR7_GD;
13980
13981	/* Paranoia. */
13982	pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13983	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
13984
13985	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
13986	AssertRCReturn(rc, rc);
13987
13988	/*
13989	* Raise #DB in the guest.
13990	*
13991	* It is important to reflect exactly what the VM-exit gave us (preserving the
13992	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
13993	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
13994	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
13995	*
13996	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
13997	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
13998	*/
13999	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14000	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14001	return VINF_SUCCESS;
14002	}
14003
14004	/*
14005	* Not a guest trap, must be a hypervisor related debug event then.
14006	* Update DR6 in case someone is interested in it.
14007	*/
14008	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
14009	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
14010	CPUMSetHyperDR6(pVCpu, uDR6);
14011
14012	return rc;
14013	}
14014
14015
14016	/**
14017	* Hacks its way around the lovely mesa driver's backdoor accesses.
14018	*
14019	* @sa hmR0SvmHandleMesaDrvGp.
14020	*/
14021	static int hmR0VmxHandleMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14022	{
14023	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
14024	RT_NOREF(pCtx);
14025
14026	/* For now we'll just skip the instruction. */
14027	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14028	}
14029
14030
14031	/**
14032	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
14033	* backdoor logging w/o checking what it is running inside.
14034	*
14035	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
14036	* backdoor port and magic numbers loaded in registers.
14037	*
14038	* @returns true if it is, false if it isn't.
14039	* @sa hmR0SvmIsMesaDrvGp.
14040	*/
14041	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14042	{
14043	/* 0xed: IN eAX,dx */
14044	uint8_t abInstr[1];
14045	if (pVmxTransient->cbInstr != sizeof(abInstr))
14046	return false;
14047
14048	/* Check that it is #GP(0). */
14049	if (pVmxTransient->uExitIntErrorCode != 0)
14050	return false;
14051
14052	/* Check magic and port. */
14053	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
14054	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
14055	if (pCtx->rax != UINT32_C(0x564d5868))
14056	return false;
14057	if (pCtx->dx != UINT32_C(0x5658))
14058	return false;
14059
14060	/* Flat ring-3 CS. */
14061	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
14062	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
14063	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
14064	if (pCtx->cs.Attr.n.u2Dpl != 3)
14065	return false;
14066	if (pCtx->cs.u64Base != 0)
14067	return false;
14068
14069	/* Check opcode. */
14070	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
14071	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
14072	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
14073	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
14074	if (RT_FAILURE(rc))
14075	return false;
14076	if (abInstr[0] != 0xed)
14077	return false;
14078
14079	return true;
14080	}
14081
14082
14083	/**
14084	* VM-exit exception handler for \#GP (General-protection exception).
14085	*
14086	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
14087	*/
14088	static VBOXSTRICTRC hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14089	{
14090	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14091	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
14092
14093	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14094	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14095	if (pVmcsInfo->RealMode.fRealOnV86Active)
14096	{ /* likely */ }
14097	else
14098	{
14099	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14100	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv \|\| pVmxTransient->fIsNestedGuest);
14101	#endif
14102	/*
14103	* If the guest is not in real-mode or we have unrestricted guest execution support, or if we are
14104	* executing a nested-guest, reflect #GP to the guest or nested-guest.
14105	*/
14106	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14107	AssertRCReturn(rc, rc);
14108	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
14109	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
14110
14111	if ( pVmxTransient->fIsNestedGuest
14112	\|\| !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
14113	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
14114	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14115	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14116	else
14117	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
14118	return rc;
14119	}
14120
14121	Assert(CPUMIsGuestInRealModeEx(pCtx));
14122	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
14123	Assert(!pVmxTransient->fIsNestedGuest);
14124
14125	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14126	AssertRCReturn(rc, rc);
14127
14128	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
14129	if (rcStrict == VINF_SUCCESS)
14130	{
14131	if (!CPUMIsGuestInRealModeEx(pCtx))
14132	{
14133	/*
14134	* The guest is no longer in real-mode, check if we can continue executing the
14135	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
14136	*/
14137	pVmcsInfo->RealMode.fRealOnV86Active = false;
14138	if (HMCanExecuteVmxGuest(pVCpu, pCtx))
14139	{
14140	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
14141	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14142	}
14143	else
14144	{
14145	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
14146	rcStrict = VINF_EM_RESCHEDULE;
14147	}
14148	}
14149	else
14150	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14151	}
14152	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14153	{
14154	rcStrict = VINF_SUCCESS;
14155	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14156	}
14157	return VBOXSTRICTRC_VAL(rcStrict);
14158	}
14159
14160
14161	/**
14162	* VM-exit exception handler wrapper for all other exceptions that are not handled
14163	* by a specific handler.
14164	*
14165	* This simply re-injects the exception back into the VM without any special
14166	* processing.
14167	*
14168	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
14169	*/
14170	static VBOXSTRICTRC hmR0VmxExitXcptOthers(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14171	{
14172	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14173
14174	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14175	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14176	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active \|\| pVmxTransient->fIsNestedGuest,
14177	("uVector=%#x u32XcptBitmap=%#X32\n",
14178	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
14179	NOREF(pVmcsInfo);
14180	#endif
14181
14182	/*
14183	* Re-inject the exception into the guest. This cannot be a double-fault condition which
14184	* would have been handled while checking exits due to event delivery.
14185	*/
14186	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
14187
14188	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14189	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
14190	AssertRCReturn(rc, rc);
14191	Log4Func(("Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
14192	#endif
14193
14194	#ifdef VBOX_WITH_STATISTICS
14195	switch (uVector)
14196	{
14197	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE); break;
14198	case X86_XCPT_DB: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB); break;
14199	case X86_XCPT_BP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP); break;
14200	case X86_XCPT_OF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
14201	case X86_XCPT_BR: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBR); break;
14202	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD); break;
14203	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
14204	case X86_XCPT_DF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDF); break;
14205	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS); break;
14206	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP); break;
14207	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS); break;
14208	case X86_XCPT_GP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP); break;
14209	case X86_XCPT_PF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF); break;
14210	case X86_XCPT_MF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF); break;
14211	case X86_XCPT_AC: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC); break;
14212	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF); break;
14213	default:
14214	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
14215	break;
14216	}
14217	#endif
14218
14219	/* We should never call this function for a page-fault, we'd need to pass on the fault address below otherwise. */
14220	Assert(!VMX_EXIT_INT_INFO_IS_XCPT_PF(pVmxTransient->uExitIntInfo));
14221	NOREF(uVector);
14222
14223	/* Re-inject the original exception into the guest. */
14224	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14225	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14226	return VINF_SUCCESS;
14227	}
14228
14229
14230	/**
14231	* VM-exit exception handler for all exceptions (except NMIs!).
14232	*
14233	* @remarks This may be called for both guests and nested-guests. Take care to not
14234	* make assumptions and avoid doing anything that is not relevant when
14235	* executing a nested-guest (e.g., Mesa driver hacks).
14236	*/
14237	static VBOXSTRICTRC hmR0VmxExitXcpt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14238	{
14239	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
14240
14241	/*
14242	* If this VM-exit occurred while delivering an event through the guest IDT, take
14243	* action based on the return code and additional hints (e.g. for page-faults)
14244	* that will be updated in the VMX transient structure.
14245	*/
14246	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14247	if (rcStrict == VINF_SUCCESS)
14248	{
14249	/*
14250	* If an exception caused a VM-exit due to delivery of an event, the original
14251	* event may have to be re-injected into the guest. We shall reinject it and
14252	* continue guest execution. However, page-fault is a complicated case and
14253	* needs additional processing done in hmR0VmxExitXcptPF().
14254	*/
14255	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
14256	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
14257	if ( !pVCpu->hm.s.Event.fPending
14258	\|\| uVector == X86_XCPT_PF)
14259	{
14260	switch (uVector)
14261	{
14262	case X86_XCPT_PF: return hmR0VmxExitXcptPF(pVCpu, pVmxTransient);
14263	case X86_XCPT_GP: return hmR0VmxExitXcptGP(pVCpu, pVmxTransient);
14264	case X86_XCPT_MF: return hmR0VmxExitXcptMF(pVCpu, pVmxTransient);
14265	case X86_XCPT_DB: return hmR0VmxExitXcptDB(pVCpu, pVmxTransient);
14266	case X86_XCPT_BP: return hmR0VmxExitXcptBP(pVCpu, pVmxTransient);
14267	case X86_XCPT_AC: return hmR0VmxExitXcptAC(pVCpu, pVmxTransient);
14268	default:
14269	return hmR0VmxExitXcptOthers(pVCpu, pVmxTransient);
14270	}
14271	}
14272	/* else: inject pending event before resuming guest execution. */
14273	}
14274	else if (rcStrict == VINF_HM_DOUBLE_FAULT)
14275	{
14276	Assert(pVCpu->hm.s.Event.fPending);
14277	rcStrict = VINF_SUCCESS;
14278	}
14279
14280	return rcStrict;
14281	}
14282	/** @} */
14283
14284
14285	/** @name VM-exit handlers.
14286	* @{
14287	*/
14288	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14289	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
14290	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14291
14292	/**
14293	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
14294	*/
14295	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14296	{
14297	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14298	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
14299	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
14300	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
14301	return VINF_SUCCESS;
14302	return VINF_EM_RAW_INTERRUPT;
14303	}
14304
14305
14306	/**
14307	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI). Conditional
14308	* VM-exit.
14309	*/
14310	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14311	{
14312	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14313	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
14314
14315	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14316	AssertRCReturn(rc, rc);
14317
14318	uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
14319	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
14320	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
14321
14322	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14323	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
14324	&& uExitIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
14325	NOREF(pVmcsInfo);
14326
14327	VBOXSTRICTRC rcStrict;
14328	switch (uExitIntType)
14329	{
14330	/*
14331	* Host physical NMIs:
14332	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
14333	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
14334	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
14335	*
14336	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
14337	* See Intel spec. 27.5.5 "Updating Non-Register State".
14338	*/
14339	case VMX_EXIT_INT_INFO_TYPE_NMI:
14340	{
14341	rcStrict = hmR0VmxExitHostNmi(pVCpu, pVmcsInfo);
14342	break;
14343	}
14344
14345	/*
14346	* Privileged software exceptions (#DB from ICEBP),
14347	* Software exceptions (#BP and #OF),
14348	* Hardware exceptions:
14349	* Process the required exceptions and resume guest execution if possible.
14350	*/
14351	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
14352	Assert(uVector == X86_XCPT_DB);
14353	RT_FALL_THRU();
14354	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
14355	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uExitIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
14356	RT_FALL_THRU();
14357	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
14358	{
14359	NOREF(uVector);
14360	rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14361	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14362	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
14363	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
14364	AssertRCReturn(rc, rc);
14365
14366	rcStrict = hmR0VmxExitXcpt(pVCpu, pVmxTransient);
14367	break;
14368	}
14369
14370	default:
14371	{
14372	pVCpu->hm.s.u32HMError = pVmxTransient->uExitIntInfo;
14373	rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
14374	AssertMsgFailed(("Invalid/unexpected VM-exit interruption info %#x\n", pVmxTransient->uExitIntInfo));
14375	break;
14376	}
14377	}
14378
14379	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
14380	return rcStrict;
14381	}
14382
14383
14384	/**
14385	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
14386	*/
14387	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14388	{
14389	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14390
14391	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
14392	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14393	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
14394	AssertRCReturn(rc, rc);
14395
14396	/* Evaluate and deliver pending events and resume guest execution. */
14397	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
14398	return VINF_SUCCESS;
14399	}
14400
14401
14402	/**
14403	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
14404	*/
14405	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14406	{
14407	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14408
14409	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14410	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
14411	{
14412	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
14413	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14414	}
14415
14416	Assert(!CPUMIsGuestNmiBlocking(pVCpu));
14417
14418	/*
14419	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
14420	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
14421	*/
14422	uint32_t fIntrState;
14423	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
14424	AssertRCReturn(rc, rc);
14425	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
14426	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
14427	{
14428	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
14429	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
14430
14431	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
14432	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
14433	AssertRCReturn(rc, rc);
14434	}
14435
14436	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
14437	rc = hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
14438	AssertRCReturn(rc, rc);
14439
14440	/* Evaluate and deliver pending events and resume guest execution. */
14441	return VINF_SUCCESS;
14442	}
14443
14444
14445	/**
14446	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
14447	*/
14448	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14449	{
14450	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14451	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14452	}
14453
14454
14455	/**
14456	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
14457	*/
14458	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14459	{
14460	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14461	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14462	}
14463
14464
14465	/**
14466	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
14467	*/
14468	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14469	{
14470	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14471
14472	/*
14473	* Get the state we need and update the exit history entry.
14474	*/
14475	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14476	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14477	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
14478	AssertRCReturn(rc, rc);
14479
14480	VBOXSTRICTRC rcStrict;
14481	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14482	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
14483	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14484	if (!pExitRec)
14485	{
14486	/*
14487	* Regular CPUID instruction execution.
14488	*/
14489	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
14490	if (rcStrict == VINF_SUCCESS)
14491	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14492	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14493	{
14494	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14495	rcStrict = VINF_SUCCESS;
14496	}
14497	}
14498	else
14499	{
14500	/*
14501	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14502	*/
14503	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14504	AssertRCReturn(rc2, rc2);
14505
14506	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
14507	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
14508
14509	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14510	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14511
14512	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14513	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14514	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14515	}
14516	return rcStrict;
14517	}
14518
14519
14520	/**
14521	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
14522	*/
14523	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14524	{
14525	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14526
14527	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14528	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
14529	AssertRCReturn(rc, rc);
14530
14531	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
14532	return VINF_EM_RAW_EMULATE_INSTR;
14533
14534	AssertMsgFailed(("hmR0VmxExitGetsec: Unexpected VM-exit when CR4.SMXE is 0.\n"));
14535	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14536	}
14537
14538
14539	/**
14540	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
14541	*/
14542	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14543	{
14544	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14545
14546	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14547	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14548	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14549	AssertRCReturn(rc, rc);
14550
14551	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
14552	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14553	{
14554	/* If we get a spurious VM-exit when TSC offsetting is enabled,
14555	we must reset offsetting on VM-entry. See @bugref{6634}. */
14556	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
14557	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14558	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14559	}
14560	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14561	{
14562	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14563	rcStrict = VINF_SUCCESS;
14564	}
14565	return rcStrict;
14566	}
14567
14568
14569	/**
14570	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
14571	*/
14572	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14573	{
14574	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14575
14576	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14577	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
14578	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14579	AssertRCReturn(rc, rc);
14580
14581	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
14582	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14583	{
14584	/* If we get a spurious VM-exit when TSC offsetting is enabled,
14585	we must reset offsetting on VM-reentry. See @bugref{6634}. */
14586	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
14587	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14588	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14589	}
14590	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14591	{
14592	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14593	rcStrict = VINF_SUCCESS;
14594	}
14595	return rcStrict;
14596	}
14597
14598
14599	/**
14600	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
14601	*/
14602	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14603	{
14604	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14605
14606	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14607	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
14608	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
14609	AssertRCReturn(rc, rc);
14610
14611	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14612	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
14613	if (RT_LIKELY(rc == VINF_SUCCESS))
14614	{
14615	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14616	Assert(pVmxTransient->cbInstr == 2);
14617	}
14618	else
14619	{
14620	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
14621	rc = VERR_EM_INTERPRETER;
14622	}
14623	return rc;
14624	}
14625
14626
14627	/**
14628	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
14629	*/
14630	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14631	{
14632	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14633
14634	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
14635	if (EMAreHypercallInstructionsEnabled(pVCpu))
14636	{
14637	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14638	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
14639	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
14640	AssertRCReturn(rc, rc);
14641
14642	/* Perform the hypercall. */
14643	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
14644	if (rcStrict == VINF_SUCCESS)
14645	{
14646	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14647	AssertRCReturn(rc, rc);
14648	}
14649	else
14650	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
14651	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
14652	\|\| RT_FAILURE(rcStrict));
14653
14654	/* If the hypercall changes anything other than guest's general-purpose registers,
14655	we would need to reload the guest changed bits here before VM-entry. */
14656	}
14657	else
14658	Log4Func(("Hypercalls not enabled\n"));
14659
14660	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
14661	if (RT_FAILURE(rcStrict))
14662	{
14663	hmR0VmxSetPendingXcptUD(pVCpu);
14664	rcStrict = VINF_SUCCESS;
14665	}
14666
14667	return rcStrict;
14668	}
14669
14670
14671	/**
14672	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
14673	*/
14674	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14675	{
14676	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14677	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
14678
14679	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14680	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14681	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14682	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
14683	AssertRCReturn(rc, rc);
14684
14685	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
14686
14687	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
14688	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14689	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14690	{
14691	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14692	rcStrict = VINF_SUCCESS;
14693	}
14694	else
14695	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) status: %Rrc\n", pVmxTransient->uExitQual,
14696	VBOXSTRICTRC_VAL(rcStrict)));
14697	return rcStrict;
14698	}
14699
14700
14701	/**
14702	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
14703	*/
14704	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14705	{
14706	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14707
14708	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14709	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
14710	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14711	AssertRCReturn(rc, rc);
14712
14713	VBOXSTRICTRC rcStrict = IEMExecDecodedMonitor(pVCpu, pVmxTransient->cbInstr);
14714	if (rcStrict == VINF_SUCCESS)
14715	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14716	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14717	{
14718	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14719	rcStrict = VINF_SUCCESS;
14720	}
14721
14722	return rcStrict;
14723	}
14724
14725
14726	/**
14727	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
14728	*/
14729	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14730	{
14731	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14732
14733	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14734	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
14735	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14736	AssertRCReturn(rc, rc);
14737
14738	VBOXSTRICTRC rcStrict = IEMExecDecodedMwait(pVCpu, pVmxTransient->cbInstr);
14739	if (RT_SUCCESS(rcStrict))
14740	{
14741	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14742	if (EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
14743	rcStrict = VINF_SUCCESS;
14744	}
14745
14746	return rcStrict;
14747	}
14748
14749
14750	/**
14751	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
14752	* VM-exit.
14753	*/
14754	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14755	{
14756	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14757	return VINF_EM_RESET;
14758	}
14759
14760
14761	/**
14762	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
14763	*/
14764	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14765	{
14766	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14767
14768	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14769	AssertRCReturn(rc, rc);
14770
14771	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
14772	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
14773	rc = VINF_SUCCESS;
14774	else
14775	rc = VINF_EM_HALT;
14776
14777	if (rc != VINF_SUCCESS)
14778	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
14779	return rc;
14780	}
14781
14782
14783	/**
14784	* VM-exit handler for instructions that result in a \#UD exception delivered to
14785	* the guest.
14786	*/
14787	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14788	{
14789	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14790	hmR0VmxSetPendingXcptUD(pVCpu);
14791	return VINF_SUCCESS;
14792	}
14793
14794
14795	/**
14796	* VM-exit handler for expiry of the VMX-preemption timer.
14797	*/
14798	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14799	{
14800	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14801
14802	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
14803	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
14804
14805	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
14806	PVM pVM = pVCpu->CTX_SUFF(pVM);
14807	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
14808	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
14809	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
14810	}
14811
14812
14813	/**
14814	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
14815	*/
14816	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14817	{
14818	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14819
14820	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14821	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14822	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
14823	AssertRCReturn(rc, rc);
14824
14825	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
14826	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
14827	: HM_CHANGED_RAISED_XCPT_MASK);
14828
14829	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14830	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
14831
14832	return rcStrict;
14833	}
14834
14835
14836	/**
14837	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
14838	*/
14839	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14840	{
14841	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14842	/** @todo Use VM-exit instruction information. */
14843	return VERR_EM_INTERPRETER;
14844	}
14845
14846
14847	/**
14848	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error
14849	* VM-exit.
14850	*/
14851	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14852	{
14853	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14854	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14855	AssertRCReturn(rc, rc);
14856
14857	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
14858	if (RT_FAILURE(rc))
14859	return rc;
14860
14861	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
14862	NOREF(uInvalidReason);
14863
14864	#ifdef VBOX_STRICT
14865	uint32_t fIntrState;
14866	RTHCUINTREG uHCReg;
14867	uint64_t u64Val;
14868	uint32_t u32Val;
14869	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
14870	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
14871	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
14872	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
14873	AssertRCReturn(rc, rc);
14874
14875	Log4(("uInvalidReason %u\n", uInvalidReason));
14876	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
14877	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
14878	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
14879	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
14880
14881	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
14882	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
14883	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
14884	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
14885	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
14886	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
14887	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
14888	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
14889	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
14890	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
14891	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
14892	{
14893	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
14894	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
14895	}
14896
14897	hmR0DumpRegs(pVCpu);
14898	#endif
14899
14900	return VERR_VMX_INVALID_GUEST_STATE;
14901	}
14902
14903	/**
14904	* VM-exit handler for all undefined/unexpected reasons. Should never happen.
14905	*/
14906	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUnexpected(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14907	{
14908	/*
14909	* Cummulative notes of all recognized but unexpected VM-exits.
14910	*
14911	* 1. This does -not- cover scenarios like like a page-fault VM-exit occurring when
14912	* nested-paging is used.
14913	*
14914	* 2. Any instruction that causes a VM-exit unconditionally (for e.g. VMXON) must be
14915	* emulated or a #UD must be raised in the guest. Therefore, we should -not- be using
14916	* this function (and thereby stop VM execution) for handling such instructions.
14917	*
14918	*
14919	* VMX_EXIT_INIT_SIGNAL:
14920	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
14921	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these
14922	* VM-exits. However, we should not receive INIT signals VM-exit while executing a VM.
14923	*
14924	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery"
14925	* See Intel spec. 29.3 "VMX Instructions" for "VMXON".
14926	* See Intel spec. "23.8 Restrictions on VMX operation".
14927	*
14928	* VMX_EXIT_SIPI:
14929	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest
14930	* activity state is used. We don't make use of it as our guests don't have direct
14931	* access to the host local APIC.
14932	*
14933	* See Intel spec. 25.3 "Other Causes of VM-exits".
14934	*
14935	* VMX_EXIT_IO_SMI:
14936	* VMX_EXIT_SMI:
14937	* This can only happen if we support dual-monitor treatment of SMI, which can be
14938	* activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer
14939	* monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in
14940	* VMX root mode or receive an SMI. If we get here, something funny is going on.
14941	*
14942	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
14943	* See Intel spec. 25.3 "Other Causes of VM-Exits"
14944	*
14945	* VMX_EXIT_ERR_MSR_LOAD:
14946	* Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected
14947	* and typically indicates a bug in the hypervisor code. We thus cannot not resume
14948	* execution.
14949	*
14950	* See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State".
14951	*
14952	* VMX_EXIT_ERR_MACHINE_CHECK:
14953	* Machine check exceptions indicates a fatal/unrecoverable hardware condition
14954	* including but not limited to system bus, ECC, parity, cache and TLB errors. A
14955	* #MC exception abort class exception is raised. We thus cannot assume a
14956	* reasonable chance of continuing any sort of execution and we bail.
14957	*
14958	* See Intel spec. 15.1 "Machine-check Architecture".
14959	* See Intel spec. 27.1 "Architectural State Before A VM Exit".
14960	*
14961	* VMX_EXIT_PML_FULL:
14962	* VMX_EXIT_VIRTUALIZED_EOI:
14963	* VMX_EXIT_APIC_WRITE:
14964	* We do not currently support any of these features and thus they are all unexpected
14965	* VM-exits.
14966	*
14967	* VMX_EXIT_GDTR_IDTR_ACCESS:
14968	* VMX_EXIT_LDTR_TR_ACCESS:
14969	* VMX_EXIT_RDRAND:
14970	* VMX_EXIT_RSM:
14971	* VMX_EXIT_VMFUNC:
14972	* VMX_EXIT_ENCLS:
14973	* VMX_EXIT_RDSEED:
14974	* VMX_EXIT_XSAVES:
14975	* VMX_EXIT_XRSTORS:
14976	* VMX_EXIT_UMWAIT:
14977	* VMX_EXIT_TPAUSE:
14978	* These VM-exits are -not- caused unconditionally by execution of the corresponding
14979	* instruction. Any VM-exit for these instructions indicate a hardware problem,
14980	* unsupported CPU modes (like SMM) or potentially corrupt VMCS controls.
14981	*
14982	* See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
14983	*/
14984	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14985	AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason));
14986	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
14987	}
14988
14989
14990	/**
14991	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
14992	*/
14993	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14994	{
14995	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14996
14997	/** @todo Optimize this: We currently drag in in the whole MSR state
14998	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
14999	* MSRs required. That would require changes to IEM and possibly CPUM too.
15000	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
15001	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15002	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
15003	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
15004	switch (idMsr)
15005	{
15006	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
15007	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
15008	}
15009
15010	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15011	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
15012	AssertRCReturn(rc, rc);
15013
15014	Log4Func(("ecx=%#RX32\n", idMsr));
15015
15016	#ifdef VBOX_STRICT
15017	Assert(!pVmxTransient->fIsNestedGuest);
15018	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
15019	{
15020	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
15021	&& idMsr != MSR_K6_EFER)
15022	{
15023	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
15024	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15025	}
15026	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
15027	{
15028	Assert(pVmcsInfo->pvMsrBitmap);
15029	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
15030	if (fMsrpm & VMXMSRPM_ALLOW_RD)
15031	{
15032	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
15033	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15034	}
15035	}
15036	}
15037	#endif
15038
15039	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
15040	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
15041	if (rcStrict == VINF_SUCCESS)
15042	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
15043	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
15044	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15045	{
15046	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15047	rcStrict = VINF_SUCCESS;
15048	}
15049	else
15050	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
15051
15052	return rcStrict;
15053	}
15054
15055
15056	/**
15057	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
15058	*/
15059	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15060	{
15061	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15062
15063	/** @todo Optimize this: We currently drag in in the whole MSR state
15064	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
15065	* MSRs required. That would require changes to IEM and possibly CPUM too.
15066	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
15067	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
15068	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
15069
15070	/*
15071	* The FS and GS base MSRs are not part of the above all-MSRs mask.
15072	* Although we don't need to fetch the base as it will be overwritten shortly, while
15073	* loading guest-state we would also load the entire segment register including limit
15074	* and attributes and thus we need to load them here.
15075	*/
15076	switch (idMsr)
15077	{
15078	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
15079	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
15080	}
15081
15082	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15083	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15084	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
15085	AssertRCReturn(rc, rc);
15086
15087	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
15088
15089	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
15090	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
15091
15092	if (rcStrict == VINF_SUCCESS)
15093	{
15094	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15095
15096	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
15097	if ( idMsr == MSR_IA32_APICBASE
15098	\|\| ( idMsr >= MSR_IA32_X2APIC_START
15099	&& idMsr <= MSR_IA32_X2APIC_END))
15100	{
15101	/*
15102	* We've already saved the APIC related guest-state (TPR) in post-run phase.
15103	* When full APIC register virtualization is implemented we'll have to make
15104	* sure APIC state is saved from the VMCS before IEM changes it.
15105	*/
15106	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
15107	}
15108	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
15109	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
15110	else if (idMsr == MSR_K6_EFER)
15111	{
15112	/*
15113	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
15114	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
15115	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
15116	*/
15117	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
15118	}
15119
15120	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not used. */
15121	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
15122	{
15123	switch (idMsr)
15124	{
15125	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
15126	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
15127	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
15128	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
15129	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
15130	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
15131	default:
15132	{
15133	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
15134	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
15135	else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
15136	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
15137	break;
15138	}
15139	}
15140	}
15141	#ifdef VBOX_STRICT
15142	else
15143	{
15144	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
15145	switch (idMsr)
15146	{
15147	case MSR_IA32_SYSENTER_CS:
15148	case MSR_IA32_SYSENTER_EIP:
15149	case MSR_IA32_SYSENTER_ESP:
15150	case MSR_K8_FS_BASE:
15151	case MSR_K8_GS_BASE:
15152	{
15153	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
15154	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15155	}
15156
15157	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
15158	default:
15159	{
15160	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
15161	{
15162	/* EFER MSR writes are always intercepted. */
15163	if (idMsr != MSR_K6_EFER)
15164	{
15165	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
15166	idMsr));
15167	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15168	}
15169	}
15170
15171	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
15172	{
15173	Assert(pVmcsInfo->pvMsrBitmap);
15174	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
15175	if (fMsrpm & VMXMSRPM_ALLOW_WR)
15176	{
15177	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
15178	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
15179	}
15180	}
15181	break;
15182	}
15183	}
15184	}
15185	#endif /* VBOX_STRICT */
15186	}
15187	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15188	{
15189	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15190	rcStrict = VINF_SUCCESS;
15191	}
15192	else
15193	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
15194
15195	return rcStrict;
15196	}
15197
15198
15199	/**
15200	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
15201	*/
15202	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15203	{
15204	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15205
15206	/** @todo The guest has likely hit a contended spinlock. We might want to
15207	* poke a schedule different guest VCPU. */
15208	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15209	if (RT_SUCCESS(rc))
15210	return VINF_EM_RAW_INTERRUPT;
15211
15212	AssertMsgFailed(("hmR0VmxExitPause: Failed to increment RIP. rc=%Rrc\n", rc));
15213	return rc;
15214	}
15215
15216
15217	/**
15218	* VM-exit handler for when the TPR value is lowered below the specified
15219	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
15220	*/
15221	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15222	{
15223	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15224	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
15225
15226	/*
15227	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
15228	* We'll re-evaluate pending interrupts and inject them before the next VM
15229	* entry so we can just continue execution here.
15230	*/
15231	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
15232	return VINF_SUCCESS;
15233	}
15234
15235
15236	/**
15237	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
15238	* VM-exit.
15239	*
15240	* @retval VINF_SUCCESS when guest execution can continue.
15241	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
15242	* @retval VERR_EM_RESCHEDULE_REM when we need to return to ring-3 due to
15243	* incompatible guest state for VMX execution (real-on-v86 case).
15244	*/
15245	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15246	{
15247	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15248	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
15249
15250	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15251	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15252	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15253	AssertRCReturn(rc, rc);
15254
15255	VBOXSTRICTRC rcStrict;
15256	PVM pVM = pVCpu->CTX_SUFF(pVM);
15257	uint64_t const uExitQual = pVmxTransient->uExitQual;
15258	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
15259	switch (uAccessType)
15260	{
15261	/*
15262	* MOV to CRx.
15263	*/
15264	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
15265	{
15266	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15267	AssertRCReturn(rc, rc);
15268
15269	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
15270	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
15271	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
15272	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
15273
15274	/*
15275	* MOV to CR3 only cause a VM-exit when one or more of the following are true:
15276	* - When nested paging isn't used.
15277	* - If the guest doesn't have paging enabled (intercept CR3 to update shadow page tables).
15278	* - We are executing in the VM debug loop.
15279	*/
15280	Assert( iCrReg != 3
15281	\|\| !pVM->hm.s.fNestedPaging
15282	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
15283	\|\| pVCpu->hm.s.fUsingDebugLoop);
15284
15285	/* MOV to CR8 writes only cause VM-exits when TPR shadow is not used. */
15286	Assert( iCrReg != 8
15287	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
15288
15289	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
15290	AssertMsg( rcStrict == VINF_SUCCESS
15291	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15292
15293	/*
15294	* This is a kludge for handling switches back to real mode when we try to use
15295	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
15296	* deal with special selector values, so we have to return to ring-3 and run
15297	* there till the selector values are V86 mode compatible.
15298	*
15299	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
15300	* latter is an alias for VINF_IEM_RAISED_XCPT which is asserted at the end of
15301	* this function.
15302	*/
15303	if ( iCrReg == 0
15304	&& rcStrict == VINF_SUCCESS
15305	&& !pVM->hm.s.vmx.fUnrestrictedGuest
15306	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
15307	&& (uOldCr0 & X86_CR0_PE)
15308	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
15309	{
15310	/** @todo Check selectors rather than returning all the time. */
15311	Assert(!pVmxTransient->fIsNestedGuest);
15312	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
15313	rcStrict = VINF_EM_RESCHEDULE_REM;
15314	}
15315	break;
15316	}
15317
15318	/*
15319	* MOV from CRx.
15320	*/
15321	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
15322	{
15323	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
15324	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
15325
15326	/*
15327	* MOV from CR3 only cause a VM-exit when one or more of the following are true:
15328	* - When nested paging isn't used.
15329	* - If the guest doesn't have paging enabled (pass guest's CR3 rather than our identity mapped CR3).
15330	* - We are executing in the VM debug loop.
15331	*/
15332	Assert( iCrReg != 3
15333	\|\| !pVM->hm.s.fNestedPaging
15334	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
15335	\|\| pVCpu->hm.s.fUsingDebugLoop);
15336
15337	/* MOV from CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
15338	Assert( iCrReg != 8
15339	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
15340
15341	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
15342	break;
15343	}
15344
15345	/*
15346	* CLTS (Clear Task-Switch Flag in CR0).
15347	*/
15348	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
15349	{
15350	rcStrict = hmR0VmxExitClts(pVCpu, pVmcsInfo, pVmxTransient->cbInstr);
15351	break;
15352	}
15353
15354	/*
15355	* LMSW (Load Machine-Status Word into CR0).
15356	* LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here.
15357	*/
15358	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW:
15359	{
15360	RTGCPTR GCPtrEffDst;
15361	uint8_t const cbInstr = pVmxTransient->cbInstr;
15362	uint16_t const uMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual);
15363	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(uExitQual);
15364	if (fMemOperand)
15365	{
15366	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
15367	AssertRCReturn(rc, rc);
15368	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
15369	}
15370	else
15371	GCPtrEffDst = NIL_RTGCPTR;
15372	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmcsInfo, cbInstr, uMsw, GCPtrEffDst);
15373	break;
15374	}
15375
15376	default:
15377	{
15378	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
15379	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
15380	}
15381	}
15382
15383	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
15384	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
15385	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15386
15387	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
15388	NOREF(pVM);
15389	return rcStrict;
15390	}
15391
15392
15393	/**
15394	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
15395	* VM-exit.
15396	*/
15397	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15398	{
15399	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15400	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
15401
15402	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15403	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15404	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15405	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15406	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
15407	\| CPUMCTX_EXTRN_EFER);
15408	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
15409	AssertRCReturn(rc, rc);
15410
15411	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
15412	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
15413	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
15414	bool const fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
15415	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
15416	bool const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
15417	bool const fDbgStepping = pVCpu->hm.s.fSingleInstruction;
15418	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
15419
15420	/*
15421	* Update exit history to see if this exit can be optimized.
15422	*/
15423	VBOXSTRICTRC rcStrict;
15424	PCEMEXITREC pExitRec = NULL;
15425	if ( !fGstStepping
15426	&& !fDbgStepping)
15427	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
15428	!fIOString
15429	? !fIOWrite
15430	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
15431	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
15432	: !fIOWrite
15433	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
15434	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
15435	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
15436	if (!pExitRec)
15437	{
15438	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
15439	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
15440
15441	uint32_t const cbValue = s_aIOSizes[uIOSize];
15442	uint32_t const cbInstr = pVmxTransient->cbInstr;
15443	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
15444	PVM pVM = pVCpu->CTX_SUFF(pVM);
15445	if (fIOString)
15446	{
15447	/*
15448	* INS/OUTS - I/O String instruction.
15449	*
15450	* Use instruction-information if available, otherwise fall back on
15451	* interpreting the instruction.
15452	*/
15453	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
15454	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
15455	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
15456	if (fInsOutsInfo)
15457	{
15458	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15459	AssertRCReturn(rc2, rc2);
15460	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
15461	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
15462	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
15463	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
15464	if (fIOWrite)
15465	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
15466	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
15467	else
15468	{
15469	/*
15470	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
15471	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
15472	* See Intel Instruction spec. for "INS".
15473	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
15474	*/
15475	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
15476	}
15477	}
15478	else
15479	rcStrict = IEMExecOne(pVCpu);
15480
15481	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
15482	fUpdateRipAlready = true;
15483	}
15484	else
15485	{
15486	/*
15487	* IN/OUT - I/O instruction.
15488	*/
15489	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
15490	uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
15491	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
15492	if (fIOWrite)
15493	{
15494	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
15495	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
15496	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
15497	&& !pCtx->eflags.Bits.u1TF)
15498	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
15499	}
15500	else
15501	{
15502	uint32_t u32Result = 0;
15503	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
15504	if (IOM_SUCCESS(rcStrict))
15505	{
15506	/* Save result of I/O IN instr. in AL/AX/EAX. */
15507	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
15508	}
15509	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
15510	&& !pCtx->eflags.Bits.u1TF)
15511	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
15512	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
15513	}
15514	}
15515
15516	if (IOM_SUCCESS(rcStrict))
15517	{
15518	if (!fUpdateRipAlready)
15519	{
15520	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
15521	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
15522	}
15523
15524	/*
15525	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
15526	* while booting Fedora 17 64-bit guest.
15527	*
15528	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
15529	*/
15530	if (fIOString)
15531	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
15532
15533	/*
15534	* If any I/O breakpoints are armed, we need to check if one triggered
15535	* and take appropriate action.
15536	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
15537	*/
15538	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
15539	AssertRCReturn(rc, rc);
15540
15541	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
15542	* execution engines about whether hyper BPs and such are pending. */
15543	uint32_t const uDr7 = pCtx->dr[7];
15544	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
15545	&& X86_DR7_ANY_RW_IO(uDr7)
15546	&& (pCtx->cr4 & X86_CR4_DE))
15547	\|\| DBGFBpIsHwIoArmed(pVM)))
15548	{
15549	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
15550
15551	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
15552	VMMRZCallRing3Disable(pVCpu);
15553	HM_DISABLE_PREEMPT(pVCpu);
15554
15555	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
15556
15557	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
15558	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
15559	{
15560	/* Raise #DB. */
15561	if (fIsGuestDbgActive)
15562	ASMSetDR6(pCtx->dr[6]);
15563	if (pCtx->dr[7] != uDr7)
15564	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
15565
15566	hmR0VmxSetPendingXcptDB(pVCpu);
15567	}
15568	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
15569	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
15570	else if ( rcStrict2 != VINF_SUCCESS
15571	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
15572	rcStrict = rcStrict2;
15573	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
15574
15575	HM_RESTORE_PREEMPT();
15576	VMMRZCallRing3Enable(pVCpu);
15577	}
15578	}
15579
15580	#ifdef VBOX_STRICT
15581	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
15582	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
15583	Assert(!fIOWrite);
15584	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
15585	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
15586	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
15587	Assert(fIOWrite);
15588	else
15589	{
15590	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
15591	* statuses, that the VMM device and some others may return. See
15592	* IOM_SUCCESS() for guidance. */
15593	AssertMsg( RT_FAILURE(rcStrict)
15594	\|\| rcStrict == VINF_SUCCESS
15595	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
15596	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
15597	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
15598	\|\| rcStrict == VINF_EM_RAW_TO_R3
15599	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15600	# endif
15601	}
15602	#endif
15603	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
15604	}
15605	else
15606	{
15607	/*
15608	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
15609	*/
15610	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15611	AssertRCReturn(rc2, rc2);
15612	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
15613	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
15614	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
15615	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15616	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
15617	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
15618
15619	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
15620	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15621
15622	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
15623	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15624	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
15625	}
15626	return rcStrict;
15627	}
15628
15629
15630	/**
15631	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
15632	* VM-exit.
15633	*/
15634	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15635	{
15636	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15637
15638	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
15639	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15640	AssertRCReturn(rc, rc);
15641	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
15642	{
15643	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15644	AssertRCReturn(rc, rc);
15645	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
15646	{
15647	uint32_t uErrCode;
15648	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
15649	{
15650	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15651	AssertRCReturn(rc, rc);
15652	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
15653	}
15654	else
15655	uErrCode = 0;
15656
15657	RTGCUINTPTR GCPtrFaultAddress;
15658	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(pVmxTransient->uIdtVectoringInfo))
15659	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
15660	else
15661	GCPtrFaultAddress = 0;
15662
15663	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15664	AssertRCReturn(rc, rc);
15665
15666	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
15667	pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
15668
15669	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo),
15670	VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo)));
15671	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
15672	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15673	}
15674	}
15675
15676	/* Fall back to the interpreter to emulate the task-switch. */
15677	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
15678	return VERR_EM_INTERPRETER;
15679	}
15680
15681
15682	/**
15683	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
15684	*/
15685	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15686	{
15687	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15688
15689	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15690	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
15691	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
15692	AssertRCReturn(rc, rc);
15693	return VINF_EM_DBG_STEPPED;
15694	}
15695
15696
15697	/**
15698	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
15699	*/
15700	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15701	{
15702	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15703	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
15704
15705	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15706	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15707	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15708	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15709	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15710	AssertRCReturn(rc, rc);
15711
15712	/*
15713	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
15714	*/
15715	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15716	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15717	{
15718	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
15719	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
15720	{
15721	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
15722	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15723	}
15724	}
15725	else
15726	{
15727	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
15728	return rcStrict;
15729	}
15730
15731	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
15732	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15733	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15734	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15735	AssertRCReturn(rc, rc);
15736
15737	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
15738	uint32_t const uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
15739	switch (uAccessType)
15740	{
15741	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
15742	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
15743	{
15744	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
15745	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
15746	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
15747
15748	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
15749	GCPhys &= PAGE_BASE_GC_MASK;
15750	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
15751	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
15752	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
15753
15754	PVM pVM = pVCpu->CTX_SUFF(pVM);
15755	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15756	rcStrict = IOMMMIOPhysHandler(pVM, pVCpu,
15757	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
15758	CPUMCTX2CORE(pCtx), GCPhys);
15759	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
15760	if ( rcStrict == VINF_SUCCESS
15761	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
15762	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
15763	{
15764	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
15765	\| HM_CHANGED_GUEST_APIC_TPR);
15766	rcStrict = VINF_SUCCESS;
15767	}
15768	break;
15769	}
15770
15771	default:
15772	{
15773	Log4Func(("uAccessType=%#x\n", uAccessType));
15774	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
15775	break;
15776	}
15777	}
15778
15779	if (rcStrict != VINF_SUCCESS)
15780	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
15781	return rcStrict;
15782	}
15783
15784
15785	/**
15786	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
15787	* VM-exit.
15788	*/
15789	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15790	{
15791	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15792	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15793
15794	/* We might get this VM-exit if the nested-guest is not intercepting MOV DRx accesses. */
15795	if (!pVmxTransient->fIsNestedGuest)
15796	{
15797	/* We should -not- get this VM-exit if the guest's debug registers were active. */
15798	if (pVmxTransient->fWasGuestDebugStateActive)
15799	{
15800	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
15801	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
15802	}
15803
15804	if ( !pVCpu->hm.s.fSingleInstruction
15805	&& !pVmxTransient->fWasHyperDebugStateActive)
15806	{
15807	Assert(!DBGFIsStepping(pVCpu));
15808	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
15809
15810	/* Don't intercept MOV DRx any more. */
15811	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
15812	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
15813	AssertRCReturn(rc, rc);
15814
15815	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
15816	VMMRZCallRing3Disable(pVCpu);
15817	HM_DISABLE_PREEMPT(pVCpu);
15818
15819	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
15820	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
15821	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
15822
15823	HM_RESTORE_PREEMPT();
15824	VMMRZCallRing3Enable(pVCpu);
15825
15826	#ifdef VBOX_WITH_STATISTICS
15827	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15828	AssertRCReturn(rc, rc);
15829	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
15830	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
15831	else
15832	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
15833	#endif
15834	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
15835	return VINF_SUCCESS;
15836	}
15837	}
15838
15839	/*
15840	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
15841	* The EFER MSR is always up-to-date.
15842	* Update the segment registers and DR7 from the CPU.
15843	*/
15844	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15845	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15846	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
15847	AssertRCReturn(rc, rc);
15848	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
15849
15850	PVM pVM = pVCpu->CTX_SUFF(pVM);
15851	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
15852	{
15853	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
15854	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
15855	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
15856	if (RT_SUCCESS(rc))
15857	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
15858	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
15859	}
15860	else
15861	{
15862	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
15863	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
15864	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
15865	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
15866	}
15867
15868	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
15869	if (RT_SUCCESS(rc))
15870	{
15871	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
15872	AssertRCReturn(rc2, rc2);
15873	return VINF_SUCCESS;
15874	}
15875	return rc;
15876	}
15877
15878
15879	/**
15880	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
15881	* Conditional VM-exit.
15882	*/
15883	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15884	{
15885	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15886	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
15887
15888	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15889	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15890	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15891	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15892	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15893	AssertRCReturn(rc, rc);
15894
15895	/*
15896	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
15897	*/
15898	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
15899	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15900	{
15901	/*
15902	* In the unlikely case where delivering an event causes an EPT misconfig (MMIO), go back to
15903	* instruction emulation to inject the original event. Otherwise, injecting the original event
15904	* using hardware-assisted VMX would would trigger the same EPT misconfig VM-exit again.
15905	*/
15906	if (!pVCpu->hm.s.Event.fPending)
15907	{ /* likely */ }
15908	else
15909	{
15910	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
15911	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15912	/** @todo NSTVMX: Think about how this should be handled. */
15913	if (pVmxTransient->fIsNestedGuest)
15914	return VERR_VMX_IPE_3;
15915	#endif
15916	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15917	}
15918	}
15919	else
15920	{
15921	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
15922	return rcStrict;
15923	}
15924
15925	/*
15926	* Get sufficent state and update the exit history entry.
15927	*/
15928	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15929	rc = hmR0VmxReadGuestPhysicalAddrVmcs(pVCpu, pVmxTransient);
15930	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
15931	AssertRCReturn(rc, rc);
15932
15933	RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr;
15934	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
15935	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
15936	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
15937	if (!pExitRec)
15938	{
15939	/*
15940	* If we succeed, resume guest execution.
15941	* If we fail in interpreting the instruction because we couldn't get the guest physical address
15942	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
15943	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
15944	* weird case. See @bugref{6043}.
15945	*/
15946	PVM pVM = pVCpu->CTX_SUFF(pVM);
15947	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15948	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
15949	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
15950	if ( rcStrict == VINF_SUCCESS
15951	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
15952	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
15953	{
15954	/* Successfully handled MMIO operation. */
15955	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
15956	\| HM_CHANGED_GUEST_APIC_TPR);
15957	rcStrict = VINF_SUCCESS;
15958	}
15959	}
15960	else
15961	{
15962	/*
15963	* Frequent exit or something needing probing. Call EMHistoryExec.
15964	*/
15965	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
15966	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
15967
15968	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
15969	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15970
15971	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
15972	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
15973	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
15974	}
15975	return rcStrict;
15976	}
15977
15978
15979	/**
15980	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
15981	* VM-exit.
15982	*/
15983	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15984	{
15985	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15986	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
15987
15988	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15989	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15990	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15991	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15992	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
15993	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
15994	AssertRCReturn(rc, rc);
15995
15996	/*
15997	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
15998	*/
15999	VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
16000	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16001	{
16002	/*
16003	* If delivery of an event causes an EPT violation (true nested #PF and not MMIO),
16004	* we shall resolve the nested #PF and re-inject the original event.
16005	*/
16006	if (pVCpu->hm.s.Event.fPending)
16007	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflectNPF);
16008	}
16009	else
16010	{
16011	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
16012	return rcStrict;
16013	}
16014
16015	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
16016	rc = hmR0VmxReadGuestPhysicalAddrVmcs(pVCpu, pVmxTransient);
16017	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
16018	AssertRCReturn(rc, rc);
16019
16020	RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr;
16021	uint64_t const uExitQual = pVmxTransient->uExitQual;
16022	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", uExitQual));
16023
16024	RTGCUINT uErrorCode = 0;
16025	if (uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
16026	uErrorCode \|= X86_TRAP_PF_ID;
16027	if (uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
16028	uErrorCode \|= X86_TRAP_PF_RW;
16029	if (uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
16030	uErrorCode \|= X86_TRAP_PF_P;
16031
16032	PVM pVM = pVCpu->CTX_SUFF(pVM);
16033	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16034	Log4Func(("at %#RX64 (%#RX64 errcode=%#x) cs:rip=%#04x:%#RX64\n", GCPhys, uExitQual, uErrorCode, pCtx->cs.Sel, pCtx->rip));
16035
16036	/*
16037	* Handle the pagefault trap for the nested shadow table.
16038	*/
16039	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
16040	rcStrict = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
16041	TRPMResetTrap(pVCpu);
16042
16043	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
16044	if ( rcStrict == VINF_SUCCESS
16045	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
16046	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
16047	{
16048	/* Successfully synced our nested page tables. */
16049	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
16050	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
16051	return VINF_SUCCESS;
16052	}
16053
16054	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
16055	return rcStrict;
16056	}
16057
16058
16059	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
16060	/**
16061	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
16062	*/
16063	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16064	{
16065	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16066
16067	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16068	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16069	\| CPUMCTX_EXTRN_HWVIRT
16070	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16071	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16072	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16073	AssertRCReturn(rc, rc);
16074
16075	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16076
16077	VMXVEXITINFO ExitInfo;
16078	RT_ZERO(ExitInfo);
16079	ExitInfo.uReason = pVmxTransient->uExitReason;
16080	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16081	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16082	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16083	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16084
16085	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
16086	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16087	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16088	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16089	{
16090	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16091	rcStrict = VINF_SUCCESS;
16092	}
16093	return rcStrict;
16094	}
16095
16096
16097	/**
16098	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
16099	*/
16100	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16101	{
16102	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16103
16104	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
16105	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
16106	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16107	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16108	AssertRCReturn(rc, rc);
16109
16110	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16111
16112	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
16113	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMLAUNCH);
16114	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
16115	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16116	{
16117	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16118	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
16119	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
16120	}
16121	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
16122	return rcStrict;
16123	}
16124
16125
16126	/**
16127	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
16128	*/
16129	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16130	{
16131	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16132
16133	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16134	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16135	\| CPUMCTX_EXTRN_HWVIRT
16136	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16137	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16138	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16139	AssertRCReturn(rc, rc);
16140
16141	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16142
16143	VMXVEXITINFO ExitInfo;
16144	RT_ZERO(ExitInfo);
16145	ExitInfo.uReason = pVmxTransient->uExitReason;
16146	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16147	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16148	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16149	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16150
16151	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
16152	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16153	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16154	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16155	{
16156	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16157	rcStrict = VINF_SUCCESS;
16158	}
16159	return rcStrict;
16160	}
16161
16162
16163	/**
16164	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
16165	*/
16166	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16167	{
16168	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16169
16170	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16171	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16172	\| CPUMCTX_EXTRN_HWVIRT
16173	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16174	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16175	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16176	AssertRCReturn(rc, rc);
16177
16178	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16179
16180	VMXVEXITINFO ExitInfo;
16181	RT_ZERO(ExitInfo);
16182	ExitInfo.uReason = pVmxTransient->uExitReason;
16183	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16184	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16185	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16186	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
16187
16188	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
16189	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16190	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16191	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16192	{
16193	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16194	rcStrict = VINF_SUCCESS;
16195	}
16196	return rcStrict;
16197	}
16198
16199
16200	/**
16201	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Conditional VM-exit.
16202	*/
16203	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16204	{
16205	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16206
16207	/*
16208	* Strictly speaking we should not get VMREAD VM-exits for shadow VMCS fields and
16209	* thus might not need to import the shadow VMCS state, it's safer just in case
16210	* code elsewhere dares look at unsynced VMCS fields.
16211	*/
16212	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16213	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16214	\| CPUMCTX_EXTRN_HWVIRT
16215	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16216	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16217	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16218	AssertRCReturn(rc, rc);
16219
16220	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16221
16222	VMXVEXITINFO ExitInfo;
16223	RT_ZERO(ExitInfo);
16224	ExitInfo.uReason = pVmxTransient->uExitReason;
16225	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16226	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16227	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16228	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16229	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
16230
16231	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
16232	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16233	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16234	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16235	{
16236	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16237	rcStrict = VINF_SUCCESS;
16238	}
16239	return rcStrict;
16240	}
16241
16242
16243	/**
16244	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
16245	*/
16246	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16247	{
16248	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16249
16250	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
16251	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
16252	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16253	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
16254	AssertRCReturn(rc, rc);
16255
16256	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16257
16258	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
16259	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMRESUME);
16260	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
16261	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16262	{
16263	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
16264	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
16265	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
16266	}
16267	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
16268	return rcStrict;
16269	}
16270
16271
16272	/**
16273	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Conditional VM-exit.
16274	*/
16275	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16276	{
16277	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16278
16279	/*
16280	* Although we should not get VMWRITE VM-exits for shadow VMCS fields, since our HM hook
16281	* gets invoked when IEM's VMWRITE instruction emulation modifies the current VMCS and it
16282	* flags re-loading the entire shadow VMCS, we should save the entire shadow VMCS here.
16283	*/
16284	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16285	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16286	\| CPUMCTX_EXTRN_HWVIRT
16287	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16288	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16289	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16290	AssertRCReturn(rc, rc);
16291
16292	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16293
16294	VMXVEXITINFO ExitInfo;
16295	RT_ZERO(ExitInfo);
16296	ExitInfo.uReason = pVmxTransient->uExitReason;
16297	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16298	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16299	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16300	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
16301	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16302
16303	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
16304	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16305	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16306	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16307	{
16308	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16309	rcStrict = VINF_SUCCESS;
16310	}
16311	return rcStrict;
16312	}
16313
16314
16315	/**
16316	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
16317	*/
16318	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16319	{
16320	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16321
16322	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16323	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
16324	\| CPUMCTX_EXTRN_HWVIRT
16325	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
16326	AssertRCReturn(rc, rc);
16327
16328	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16329
16330	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
16331	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16332	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
16333	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16334	{
16335	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16336	rcStrict = VINF_SUCCESS;
16337	}
16338	return rcStrict;
16339	}
16340
16341
16342	/**
16343	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
16344	*/
16345	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16346	{
16347	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16348
16349	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16350	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16351	\| CPUMCTX_EXTRN_HWVIRT
16352	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16353	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16354	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16355	AssertRCReturn(rc, rc);
16356
16357	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16358
16359	VMXVEXITINFO ExitInfo;
16360	RT_ZERO(ExitInfo);
16361	ExitInfo.uReason = pVmxTransient->uExitReason;
16362	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16363	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16364	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16365	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16366
16367	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
16368	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16369	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
16370	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16371	{
16372	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16373	rcStrict = VINF_SUCCESS;
16374	}
16375	return rcStrict;
16376	}
16377
16378
16379	/**
16380	* VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
16381	*/
16382	HMVMX_EXIT_DECL hmR0VmxExitInvvpid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16383	{
16384	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16385
16386	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16387	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
16388	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
16389	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16390	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16391	AssertRCReturn(rc, rc);
16392
16393	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
16394
16395	VMXVEXITINFO ExitInfo;
16396	RT_ZERO(ExitInfo);
16397	ExitInfo.uReason = pVmxTransient->uExitReason;
16398	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16399	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
16400	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16401	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
16402
16403	VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
16404	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
16405	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
16406	else if (rcStrict == VINF_IEM_RAISED_XCPT)
16407	{
16408	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16409	rcStrict = VINF_SUCCESS;
16410	}
16411	return rcStrict;
16412	}
16413	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
16414	/** @} */
16415
16416
16417	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
16418	/** @name Nested-guest VM-exit handlers.
16419	* @{
16420	*/
16421	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16422	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- Nested-guest VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16423	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
16424
16425	/**
16426	* Nested-guest VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
16427	* Conditional VM-exit.
16428	*/
16429	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmiNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16430	{
16431	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16432
16433	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
16434	AssertRCReturn(rc, rc);
16435
16436	uint64_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
16437	uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
16438	Assert(VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo));
16439
16440	switch (uExitIntType)
16441	{
16442	/*
16443	* Physical NMIs:
16444	* We shouldn't direct host physical NMIs to the nested-guest. Dispatch it to the host.
16445	*/
16446	case VMX_EXIT_INT_INFO_TYPE_NMI:
16447	return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
16448
16449	/*
16450	* Hardware exceptions,
16451	* Software exceptions,
16452	* Privileged software exceptions:
16453	* Figure out if the exception must be delivered to the guest or the nested-guest.
16454	*/
16455	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
16456	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
16457	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
16458	{
16459	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
16460	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16461	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16462	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16463	AssertRCReturn(rc, rc);
16464
16465	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
16466	bool const fIntercept = CPUMIsGuestVmxXcptInterceptSet(pVCpu, pCtx, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo),
16467	pVmxTransient->uExitIntErrorCode);
16468	if (fIntercept)
16469	{
16470	/* Exit qualification is required for debug and page-fault exceptions. */
16471	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16472	AssertRCReturn(rc, rc);
16473
16474	/*
16475	* For VM-exits due to software exceptions (those generated by INT3 or INTO) and privileged
16476	* software exceptions (those generated by INT1/ICEBP) we need to supply the VM-exit instruction
16477	* length. However, if delivery of a software interrupt, software exception or privileged
16478	* software exception causes a VM-exit, that too provides the VM-exit instruction length.
16479	*/
16480	VMXVEXITINFO ExitInfo;
16481	RT_ZERO(ExitInfo);
16482	ExitInfo.uReason = pVmxTransient->uExitReason;
16483	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16484	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16485
16486	VMXVEXITEVENTINFO ExitEventInfo;
16487	RT_ZERO(ExitEventInfo);
16488	ExitEventInfo.uExitIntInfo = pVmxTransient->uExitIntInfo;
16489	ExitEventInfo.uExitIntErrCode = pVmxTransient->uExitIntErrorCode;
16490	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16491	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16492
16493	#ifdef DEBUG_ramshankar
16494	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
16495	Log4Func(("cs:rip=%#04x:%#RX64 %s err_code=%#x exit_qual=%#RX64\n", pCtx->cs.Sel, pCtx->rip,
16496	VMX_EXIT_INT_INFO_IS_XCPT_PF(pVmxTransient->uExitIntInfo) ? "#PF" : "Unk",
16497	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual));
16498	Log4Func(("idt_info=%#RX64 (%s) idt_errcode=%#RX32\n", pVmxTransient->uIdtVectoringInfo,
16499	VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo) ? "Valid" : "Invalid",
16500	pVmxTransient->uIdtVectoringErrorCode));
16501	#endif
16502	return IEMExecVmxVmexitXcpt(pVCpu, &ExitInfo, &ExitEventInfo);
16503	}
16504
16505	/* Nested paging is currently a requirement, otherwise we would need to handle shadow #PFs in hmR0VmxExitXcptPF. */
16506	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
16507	return hmR0VmxExitXcpt(pVCpu, pVmxTransient);
16508	}
16509
16510	/*
16511	* Software interrupts:
16512	* VM-exits cannot be caused by software interrupts.
16513	*
16514	* External interrupts:
16515	* This should only happen when "acknowledge external interrupts on VM-exit"
16516	* control is set. However, we never set this when executing a guest or
16517	* nested-guest. For nested-guests it is emulated while injecting interrupts into
16518	* the guest.
16519	*/
16520	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
16521	case VMX_EXIT_INT_INFO_TYPE_EXT_INT:
16522	default:
16523	{
16524	pVCpu->hm.s.u32HMError = pVmxTransient->uExitIntInfo;
16525	return VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
16526	}
16527	}
16528	}
16529
16530
16531	/**
16532	* Nested-guest VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT).
16533	* Unconditional VM-exit.
16534	*/
16535	HMVMX_EXIT_DECL hmR0VmxExitTripleFaultNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16536	{
16537	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16538	return IEMExecVmxVmexitTripleFault(pVCpu);
16539	}
16540
16541
16542	/**
16543	* Nested-guest VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
16544	*/
16545	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindowNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16546	{
16547	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16548
16549	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INT_WINDOW_EXIT))
16550	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16551	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
16552	}
16553
16554
16555	/**
16556	* Nested-guest VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
16557	*/
16558	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindowNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16559	{
16560	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16561
16562	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_NMI_WINDOW_EXIT))
16563	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
16564	return hmR0VmxExitIntWindow(pVCpu, pVmxTransient);
16565	}
16566
16567
16568	/**
16569	* Nested-guest VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH).
16570	* Unconditional VM-exit.
16571	*/
16572	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitchNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16573	{
16574	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16575
16576	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16577	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16578	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
16579	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
16580	AssertRCReturn(rc, rc);
16581
16582	VMXVEXITINFO ExitInfo;
16583	RT_ZERO(ExitInfo);
16584	ExitInfo.uReason = pVmxTransient->uExitReason;
16585	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16586	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16587
16588	VMXVEXITEVENTINFO ExitEventInfo;
16589	RT_ZERO(ExitEventInfo);
16590	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
16591	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
16592	return IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
16593	}
16594
16595
16596	/**
16597	* Nested-guest VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
16598	*/
16599	HMVMX_EXIT_DECL hmR0VmxExitHltNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16600	{
16601	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16602
16603	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
16604	{
16605	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16606	AssertRCReturn(rc, rc);
16607	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16608	}
16609	return hmR0VmxExitHlt(pVCpu, pVmxTransient);
16610	}
16611
16612
16613	/**
16614	* Nested-guest VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
16615	*/
16616	HMVMX_EXIT_DECL hmR0VmxExitInvlpgNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16617	{
16618	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16619
16620	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
16621	{
16622	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16623	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16624	AssertRCReturn(rc, rc);
16625
16626	VMXVEXITINFO ExitInfo;
16627	RT_ZERO(ExitInfo);
16628	ExitInfo.uReason = pVmxTransient->uExitReason;
16629	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16630	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16631	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16632	}
16633	return hmR0VmxExitInvlpg(pVCpu, pVmxTransient);
16634	}
16635
16636
16637	/**
16638	* Nested-guest VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
16639	*/
16640	HMVMX_EXIT_DECL hmR0VmxExitRdpmcNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16641	{
16642	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16643
16644	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDPMC_EXIT))
16645	{
16646	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16647	AssertRCReturn(rc, rc);
16648	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16649	}
16650	return hmR0VmxExitRdpmc(pVCpu, pVmxTransient);
16651	}
16652
16653
16654	/**
16655	* Nested-guest VM-exit handler for VMREAD (VMX_EXIT_VMREAD) and VMWRITE
16656	* (VMX_EXIT_VMWRITE). Conditional VM-exit.
16657	*/
16658	HMVMX_EXIT_DECL hmR0VmxExitVmreadVmwriteNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16659	{
16660	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16661
16662	Assert( pVmxTransient->uExitReason == VMX_EXIT_VMREAD
16663	\|\| pVmxTransient->uExitReason == VMX_EXIT_VMWRITE);
16664
16665	int rc = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16666	AssertRCReturn(rc, rc);
16667
16668	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
16669	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
16670	uint64_t u64VmcsField = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
16671
16672	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
16673	if (!CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
16674	u64VmcsField &= UINT64_C(0xffffffff);
16675
16676	if (CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, pVmxTransient->uExitReason, u64VmcsField))
16677	{
16678	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16679	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16680	AssertRCReturn(rc, rc);
16681
16682	VMXVEXITINFO ExitInfo;
16683	RT_ZERO(ExitInfo);
16684	ExitInfo.uReason = pVmxTransient->uExitReason;
16685	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16686	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16687	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16688	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16689	}
16690
16691	if (pVmxTransient->uExitReason == VMX_EXIT_VMREAD)
16692	return hmR0VmxExitVmread(pVCpu, pVmxTransient);
16693	return hmR0VmxExitVmwrite(pVCpu, pVmxTransient);
16694	}
16695
16696
16697	/**
16698	* Nested-guest VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
16699	*/
16700	HMVMX_EXIT_DECL hmR0VmxExitRdtscNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16701	{
16702	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16703
16704	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
16705	{
16706	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16707	AssertRCReturn(rc, rc);
16708	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16709	}
16710
16711	return hmR0VmxExitRdtsc(pVCpu, pVmxTransient);
16712	}
16713
16714
16715	/**
16716	* Nested-guest VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX).
16717	* Conditional VM-exit.
16718	*/
16719	HMVMX_EXIT_DECL hmR0VmxExitMovCRxNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16720	{
16721	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16722
16723	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16724	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16725	AssertRCReturn(rc, rc);
16726
16727	VBOXSTRICTRC rcStrict;
16728	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual);
16729	switch (uAccessType)
16730	{
16731	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
16732	{
16733	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
16734	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
16735	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
16736	uint64_t const uNewCrX = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
16737
16738	bool fIntercept;
16739	switch (iCrReg)
16740	{
16741	case 0:
16742	case 4:
16743	fIntercept = CPUMIsGuestVmxMovToCr0Cr4InterceptSet(pVCpu, &pVCpu->cpum.GstCtx, iCrReg, uNewCrX);
16744	break;
16745
16746	case 3:
16747	fIntercept = CPUMIsGuestVmxMovToCr3InterceptSet(pVCpu, uNewCrX);
16748	break;
16749
16750	case 8:
16751	fIntercept = CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_CR8_LOAD_EXIT);
16752	break;
16753
16754	default:
16755	fIntercept = false;
16756	break;
16757	}
16758	if (fIntercept)
16759	{
16760	VMXVEXITINFO ExitInfo;
16761	RT_ZERO(ExitInfo);
16762	ExitInfo.uReason = pVmxTransient->uExitReason;
16763	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16764	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16765	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16766	}
16767	else
16768	rcStrict = hmR0VmxExitMovToCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
16769	break;
16770	}
16771
16772	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
16773	{
16774	/*
16775	* CR0/CR4 reads do not cause VM-exits, the read-shadow is used (subject to masking).
16776	* CR2 reads do not cause a VM-exit.
16777	* CR3 reads cause a VM-exit depending on the "CR3 store exiting" control.
16778	* CR8 reads cause a VM-exit depending on the "CR8 store exiting" control.
16779	*/
16780	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
16781	if ( iCrReg == 3
16782	\|\| iCrReg == 8)
16783	{
16784	static const uint32_t s_auCrXReadIntercepts[] = { 0, 0, 0, VMX_PROC_CTLS_CR3_STORE_EXIT, 0,
16785	0, 0, 0, VMX_PROC_CTLS_CR8_STORE_EXIT };
16786	uint32_t const uIntercept = s_auCrXReadIntercepts[iCrReg];
16787	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, uIntercept))
16788	{
16789	VMXVEXITINFO ExitInfo;
16790	RT_ZERO(ExitInfo);
16791	ExitInfo.uReason = pVmxTransient->uExitReason;
16792	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16793	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16794	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16795	}
16796	else
16797	{
16798	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
16799	rcStrict = hmR0VmxExitMovFromCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, iGReg, iCrReg);
16800	}
16801	}
16802	else
16803	{
16804	AssertMsgFailed(("MOV from CR%d VM-exit must not happen\n", iCrReg));
16805	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, iCrReg);
16806	}
16807	break;
16808	}
16809
16810	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
16811	{
16812	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
16813	Assert(pVmcsNstGst);
16814	uint64_t const uGstHostMask = pVmcsNstGst->u64Cr0Mask.u;
16815	uint64_t const uReadShadow = pVmcsNstGst->u64Cr0ReadShadow.u;
16816	if ( (uGstHostMask & X86_CR0_TS)
16817	&& (uReadShadow & X86_CR0_TS))
16818	{
16819	VMXVEXITINFO ExitInfo;
16820	RT_ZERO(ExitInfo);
16821	ExitInfo.uReason = pVmxTransient->uExitReason;
16822	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16823	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16824	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16825	}
16826	else
16827	rcStrict = hmR0VmxExitClts(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr);
16828	break;
16829	}
16830
16831	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
16832	{
16833	RTGCPTR GCPtrEffDst;
16834	uint16_t const uNewMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(pVmxTransient->uExitQual);
16835	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(pVmxTransient->uExitQual);
16836	if (fMemOperand)
16837	{
16838	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
16839	AssertRCReturn(rc, rc);
16840	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
16841	}
16842	else
16843	GCPtrEffDst = NIL_RTGCPTR;
16844
16845	if (CPUMIsGuestVmxLmswInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, uNewMsw))
16846	{
16847	VMXVEXITINFO ExitInfo;
16848	RT_ZERO(ExitInfo);
16849	ExitInfo.uReason = pVmxTransient->uExitReason;
16850	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16851	ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
16852	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16853	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16854	}
16855	else
16856	rcStrict = hmR0VmxExitLmsw(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbInstr, uNewMsw, GCPtrEffDst);
16857	break;
16858	}
16859
16860	default:
16861	{
16862	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
16863	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
16864	}
16865	}
16866
16867	if (rcStrict == VINF_IEM_RAISED_XCPT)
16868	{
16869	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
16870	rcStrict = VINF_SUCCESS;
16871	}
16872	return rcStrict;
16873	}
16874
16875
16876	/**
16877	* Nested-guest VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX).
16878	* Conditional VM-exit.
16879	*/
16880	HMVMX_EXIT_DECL hmR0VmxExitMovDRxNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16881	{
16882	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16883
16884	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MOV_DR_EXIT))
16885	{
16886	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16887	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16888	AssertRCReturn(rc, rc);
16889
16890	VMXVEXITINFO ExitInfo;
16891	RT_ZERO(ExitInfo);
16892	ExitInfo.uReason = pVmxTransient->uExitReason;
16893	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16894	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16895	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16896	}
16897	return hmR0VmxExitMovDRx(pVCpu, pVmxTransient);
16898	}
16899
16900
16901	/**
16902	* Nested-guest VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR).
16903	* Conditional VM-exit.
16904	*/
16905	HMVMX_EXIT_DECL hmR0VmxExitIoInstrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16906	{
16907	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16908
16909	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
16910	AssertRCReturn(rc, rc);
16911
16912	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
16913	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
16914	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
16915
16916	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
16917	uint8_t const cbAccess = s_aIOSizes[uIOSize];
16918	if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
16919	{
16920	/*
16921	* IN/OUT instruction:
16922	* - Provides VM-exit instruction length.
16923	*
16924	* INS/OUTS instruction:
16925	* - Provides VM-exit instruction length.
16926	* - Provides Guest-linear address.
16927	* - Optionally provides VM-exit instruction info (depends on CPU feature).
16928	*/
16929	PVM pVM = pVCpu->CTX_SUFF(pVM);
16930	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16931	AssertRCReturn(rc, rc);
16932
16933	/* Make sure we don't use stale/uninitialized VMX-transient info. below. */
16934	pVmxTransient->ExitInstrInfo.u = 0;
16935	pVmxTransient->uGuestLinearAddr = 0;
16936
16937	bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
16938	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
16939	if (fIOString)
16940	{
16941	rc \|= hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
16942	if (fVmxInsOutsInfo)
16943	{
16944	Assert(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
16945	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
16946	}
16947	}
16948	AssertRCReturn(rc, rc);
16949
16950	VMXVEXITINFO ExitInfo;
16951	RT_ZERO(ExitInfo);
16952	ExitInfo.uReason = pVmxTransient->uExitReason;
16953	ExitInfo.cbInstr = pVmxTransient->cbInstr;
16954	ExitInfo.u64Qual = pVmxTransient->uExitQual;
16955	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
16956	ExitInfo.u64GuestLinearAddr = pVmxTransient->uGuestLinearAddr;
16957	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
16958	}
16959	return hmR0VmxExitIoInstr(pVCpu, pVmxTransient);
16960	}
16961
16962
16963	/**
16964	* Nested-guest VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
16965	*/
16966	HMVMX_EXIT_DECL hmR0VmxExitRdmsrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16967	{
16968	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16969
16970	uint32_t fMsrpm;
16971	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
16972	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
16973	else
16974	fMsrpm = VMXMSRPM_EXIT_RD;
16975
16976	if (fMsrpm & VMXMSRPM_EXIT_RD)
16977	{
16978	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
16979	AssertRCReturn(rc, rc);
16980	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
16981	}
16982	return hmR0VmxExitRdmsr(pVCpu, pVmxTransient);
16983	}
16984
16985
16986	/**
16987	* Nested-guest VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
16988	*/
16989	HMVMX_EXIT_DECL hmR0VmxExitWrmsrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
16990	{
16991	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
16992
16993	uint32_t fMsrpm;
16994	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
16995	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), pVCpu->cpum.GstCtx.ecx);
16996	else
16997	fMsrpm = VMXMSRPM_EXIT_WR;
16998
16999	if (fMsrpm & VMXMSRPM_EXIT_WR)
17000	{
17001	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17002	AssertRCReturn(rc, rc);
17003	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17004	}
17005	return hmR0VmxExitWrmsr(pVCpu, pVmxTransient);
17006	}
17007
17008
17009	/**
17010	* Nested-guest VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
17011	*/
17012	HMVMX_EXIT_DECL hmR0VmxExitMwaitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17013	{
17014	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17015
17016	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MWAIT_EXIT))
17017	{
17018	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17019	AssertRCReturn(rc, rc);
17020	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17021	}
17022	return hmR0VmxExitMwait(pVCpu, pVmxTransient);
17023	}
17024
17025
17026	/**
17027	* Nested-guest VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional
17028	* VM-exit.
17029	*/
17030	HMVMX_EXIT_DECL hmR0VmxExitMtfNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17031	{
17032	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17033
17034	/** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */
17035	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
17036	}
17037
17038
17039	/**
17040	* Nested-guest VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
17041	*/
17042	HMVMX_EXIT_DECL hmR0VmxExitMonitorNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17043	{
17044	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17045
17046	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MONITOR_EXIT))
17047	{
17048	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17049	AssertRCReturn(rc, rc);
17050	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17051	}
17052	return hmR0VmxExitMonitor(pVCpu, pVmxTransient);
17053	}
17054
17055
17056	/**
17057	* Nested-guest VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
17058	*/
17059	HMVMX_EXIT_DECL hmR0VmxExitPauseNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17060	{
17061	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17062
17063	/** @todo NSTVMX: Think about this more. Does the outer guest need to intercept
17064	* PAUSE when executing a nested-guest? If it does not, we would not need
17065	* to check for the intercepts here. Just call VM-exit... */
17066
17067	/* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */
17068	if ( CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT)
17069	\|\| CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
17070	{
17071	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17072	AssertRCReturn(rc, rc);
17073	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17074	}
17075	return hmR0VmxExitPause(pVCpu, pVmxTransient);
17076	}
17077
17078
17079	/**
17080	* Nested-guest VM-exit handler for when the TPR value is lowered below the
17081	* specified threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
17082	*/
17083	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThresholdNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17084	{
17085	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17086
17087	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_TPR_SHADOW))
17088	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
17089	return hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient);
17090	}
17091
17092
17093	/**
17094	* Nested-guest VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional
17095	* VM-exit.
17096	*/
17097	HMVMX_EXIT_DECL hmR0VmxExitApicAccessNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17098	{
17099	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17100
17101	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17102	rc \|= hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
17103	rc \|= hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
17104	AssertRCReturn(rc, rc);
17105
17106	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS));
17107	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17108	AssertRCReturn(rc, rc);
17109
17110	VMXVEXITINFO ExitInfo;
17111	RT_ZERO(ExitInfo);
17112	ExitInfo.uReason = pVmxTransient->uExitReason;
17113	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17114	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17115
17116	VMXVEXITEVENTINFO ExitEventInfo;
17117	RT_ZERO(ExitEventInfo);
17118	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
17119	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
17120	return IEMExecVmxVmexitApicAccess(pVCpu, &ExitInfo, &ExitEventInfo);
17121	}
17122
17123
17124	/**
17125	* Nested-guest VM-exit handler for APIC write emulation (VMX_EXIT_APIC_WRITE).
17126	* Conditional VM-exit.
17127	*/
17128	HMVMX_EXIT_DECL hmR0VmxExitApicWriteNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17129	{
17130	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17131
17132	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_APIC_REG_VIRT));
17133	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17134	AssertRCReturn(rc, rc);
17135
17136	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
17137	}
17138
17139
17140	/**
17141	* Nested-guest VM-exit handler for virtualized EOI (VMX_EXIT_VIRTUALIZED_EOI).
17142	* Conditional VM-exit.
17143	*/
17144	HMVMX_EXIT_DECL hmR0VmxExitVirtEoiNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17145	{
17146	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17147
17148	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_INT_DELIVERY));
17149	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17150	AssertRCReturn(rc, rc);
17151
17152	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
17153	}
17154
17155
17156	/**
17157	* Nested-guest VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
17158	*/
17159	HMVMX_EXIT_DECL hmR0VmxExitRdtscpNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17160	{
17161	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17162
17163	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
17164	{
17165	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_RDTSCP));
17166	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17167	AssertRCReturn(rc, rc);
17168	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17169	}
17170	return hmR0VmxExitRdtscp(pVCpu, pVmxTransient);
17171	}
17172
17173
17174	/**
17175	* Nested-guest VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
17176	*/
17177	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvdNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17178	{
17179	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17180
17181	if (CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
17182	{
17183	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17184	AssertRCReturn(rc, rc);
17185	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17186	}
17187	return hmR0VmxExitWbinvd(pVCpu, pVmxTransient);
17188	}
17189
17190
17191	/**
17192	* Nested-guest VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
17193	*/
17194	HMVMX_EXIT_DECL hmR0VmxExitInvpcidNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17195	{
17196	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17197
17198	if (CPUMIsGuestVmxProcCtlsSet(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
17199	{
17200	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_INVPCID));
17201	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17202	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17203	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17204	AssertRCReturn(rc, rc);
17205
17206	VMXVEXITINFO ExitInfo;
17207	RT_ZERO(ExitInfo);
17208	ExitInfo.uReason = pVmxTransient->uExitReason;
17209	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17210	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17211	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17212	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17213	}
17214	return hmR0VmxExitInvpcid(pVCpu, pVmxTransient);
17215	}
17216
17217
17218	/**
17219	* Nested-guest VM-exit handler for invalid-guest state
17220	* (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error VM-exit.
17221	*/
17222	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestStateNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17223	{
17224	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17225
17226	/*
17227	* Currently this should never happen because we fully emulate VMLAUNCH/VMRESUME in IEM.
17228	* So if it does happen, it indicates a bug possibly in the hardware-assisted VMX code.
17229	* Handle it like it's in an invalid guest state of the outer guest.
17230	*
17231	* When the fast path is implemented, this should be changed to cause the corresponding
17232	* nested-guest VM-exit.
17233	*/
17234	return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
17235	}
17236
17237
17238	/**
17239	* Nested-guest VM-exit handler for instructions that cause VM-exits uncondtionally
17240	* and only provide the instruction length.
17241	*
17242	* Unconditional VM-exit.
17243	*/
17244	HMVMX_EXIT_DECL hmR0VmxExitInstrNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17245	{
17246	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17247
17248	#ifdef VBOX_STRICT
17249	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
17250	switch (pVmxTransient->uExitReason)
17251	{
17252	case VMX_EXIT_ENCLS:
17253	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_ENCLS_EXIT));
17254	break;
17255
17256	case VMX_EXIT_VMFUNC:
17257	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_VMFUNC));
17258	break;
17259	}
17260	#endif
17261
17262	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17263	AssertRCReturn(rc, rc);
17264	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbInstr);
17265	}
17266
17267
17268	/**
17269	* Nested-guest VM-exit handler for instructions that provide instruction length as
17270	* well as more information.
17271	*
17272	* Unconditional VM-exit.
17273	*/
17274	HMVMX_EXIT_DECL hmR0VmxExitInstrWithInfoNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
17275	{
17276	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
17277
17278	#ifdef VBOX_STRICT
17279	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
17280	switch (pVmxTransient->uExitReason)
17281	{
17282	case VMX_EXIT_GDTR_IDTR_ACCESS:
17283	case VMX_EXIT_LDTR_TR_ACCESS:
17284	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_DESC_TABLE_EXIT));
17285	break;
17286
17287	case VMX_EXIT_RDRAND:
17288	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDRAND_EXIT));
17289	break;
17290
17291	case VMX_EXIT_RDSEED:
17292	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_RDSEED_EXIT));
17293	break;
17294
17295	case VMX_EXIT_XSAVES:
17296	case VMX_EXIT_XRSTORS:
17297	/** @todo NSTVMX: Verify XSS-bitmap. */
17298	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_XSAVES_XRSTORS));
17299	break;
17300
17301	case VMX_EXIT_UMWAIT:
17302	case VMX_EXIT_TPAUSE:
17303	Assert(CPUMIsGuestVmxProcCtlsSet(pVCpu, pCtx, VMX_PROC_CTLS_RDTSC_EXIT));
17304	Assert(CPUMIsGuestVmxProcCtls2Set(pVCpu, pCtx, VMX_PROC_CTLS2_USER_WAIT_PAUSE));
17305	break;
17306	}
17307	#endif
17308
17309	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
17310	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
17311	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
17312	AssertRCReturn(rc, rc);
17313
17314	VMXVEXITINFO ExitInfo;
17315	RT_ZERO(ExitInfo);
17316	ExitInfo.uReason = pVmxTransient->uExitReason;
17317	ExitInfo.cbInstr = pVmxTransient->cbInstr;
17318	ExitInfo.u64Qual = pVmxTransient->uExitQual;
17319	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
17320	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
17321	}
17322
17323	/** @} */
17324	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
17325

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 80094

Download in other formats: