VMXAllTemplate.cpp.h@ 93582

Last change on this file since 93582 was 93582, checked in by vboxsync, 3 years ago
VMM/{VMXAllTemplate.cpp.h,NEMR3Native-darwin.cpp}: Temporarily intercept all CR3 accesses to investigate an issue with 32bit guests where we seem to end up with an invalid page table root address under certain circumstances, bugref:9044
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 430.8 KB

Line
1	/* $Id: VMXAllTemplate.cpp.h 93582 2022-02-03 14:15:58Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Code template for our own hypervisor and the NEM darwin backend using Apple's Hypervisor.framework.
4	*/
5
6	/*
7	* Copyright (C) 2012-2022 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Defined Constants And Macros *
21	*********************************************************************************************************************************/
22	#if !defined(VMX_VMCS_WRITE_16) \|\| !defined(VMX_VMCS_WRITE_32) \|\| !defined(VMX_VMCS_WRITE_64) \|\| !defined(VMX_VMCS_WRITE_64)
23	# error "At least one of the VMX_VMCS_WRITE_16, VMX_VMCS_WRITE_32, VMX_VMCS_WRITE_64 or VMX_VMCS_WRITE_64 is missing"
24	#endif
25
26
27	#if !defined(VMX_VMCS_READ_16) \|\| !defined(VMX_VMCS_READ_32) \|\| !defined(VMX_VMCS_READ_64) \|\| !defined(VMX_VMCS_READ_64)
28	# error "At least one of the VMX_VMCS_READ_16, VMX_VMCS_READ_32, VMX_VMCS_READ_64 or VMX_VMCS_READ_64 is missing"
29	#endif
30
31
32	/** Use the function table. */
33	#define HMVMX_USE_FUNCTION_TABLE
34
35	/** Determine which tagged-TLB flush handler to use. */
36	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
37	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
38	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
39	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
40
41	/**
42	* Flags to skip redundant reads of some common VMCS fields that are not part of
43	* the guest-CPU or VCPU state but are needed while handling VM-exits.
44	*/
45	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
46	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
47	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
48	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
49	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
50	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
51	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
52	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
53	#define HMVMX_READ_GUEST_PHYSICAL_ADDR RT_BIT_32(8)
54	#define HMVMX_READ_GUEST_PENDING_DBG_XCPTS RT_BIT_32(9)
55
56	/** All the VMCS fields required for processing of exception/NMI VM-exits. */
57	#define HMVMX_READ_XCPT_INFO ( HMVMX_READ_EXIT_INTERRUPTION_INFO \
58	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE \
59	\| HMVMX_READ_EXIT_INSTR_LEN \
60	\| HMVMX_READ_IDT_VECTORING_INFO \
61	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE)
62
63	/** Assert that all the given fields have been read from the VMCS. */
64	#ifdef VBOX_STRICT
65	# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) \
66	do { \
67	uint32_t const fVmcsFieldRead = ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead); \
68	Assert((fVmcsFieldRead & (a_fReadFields)) == (a_fReadFields)); \
69	} while (0)
70	#else
71	# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) do { } while (0)
72	#endif
73
74	/**
75	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
76	* guest using hardware-assisted VMX.
77	*
78	* This excludes state like GPRs (other than RSP) which are always are
79	* swapped and restored across the world-switch and also registers like EFER,
80	* MSR which cannot be modified by the guest without causing a VM-exit.
81	*/
82	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
83	\| CPUMCTX_EXTRN_RFLAGS \
84	\| CPUMCTX_EXTRN_RSP \
85	\| CPUMCTX_EXTRN_SREG_MASK \
86	\| CPUMCTX_EXTRN_TABLE_MASK \
87	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
88	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
89	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
90	\| CPUMCTX_EXTRN_TSC_AUX \
91	\| CPUMCTX_EXTRN_OTHER_MSRS \
92	\| CPUMCTX_EXTRN_CR0 \
93	\| CPUMCTX_EXTRN_CR3 \
94	\| CPUMCTX_EXTRN_CR4 \
95	\| CPUMCTX_EXTRN_DR7 \
96	\| CPUMCTX_EXTRN_HWVIRT \
97	\| CPUMCTX_EXTRN_INHIBIT_INT \
98	\| CPUMCTX_EXTRN_INHIBIT_NMI)
99
100	/**
101	* Exception bitmap mask for real-mode guests (real-on-v86).
102	*
103	* We need to intercept all exceptions manually except:
104	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
105	* due to bugs in Intel CPUs.
106	* - \#PF need not be intercepted even in real-mode if we have nested paging
107	* support.
108	*/
109	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
110	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
111	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
112	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
113	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
114	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
115	\| RT_BIT(X86_XCPT_XF))
116
117	/** Maximum VM-instruction error number. */
118	#define HMVMX_INSTR_ERROR_MAX 28
119
120	/** Profiling macro. */
121	#ifdef HM_PROFILE_EXIT_DISPATCH
122	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitDispatch, ed)
123	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitDispatch, ed)
124	#else
125	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
126	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
127	#endif
128
129	#ifndef IN_NEM_DARWIN
130	/** Assert that preemption is disabled or covered by thread-context hooks. */
131	# define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
132	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
133
134	/** Assert that we haven't migrated CPUs when thread-context hooks are not
135	* used. */
136	# define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
137	\|\| (a_pVCpu)->hmr0.s.idEnteredCpu == RTMpCpuId(), \
138	("Illegal migration! Entered on CPU %u Current %u\n", \
139	(a_pVCpu)->hmr0.s.idEnteredCpu, RTMpCpuId()))
140	#else
141	# define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) do { } while (0)
142	# define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) do { } while (0)
143	#endif
144
145	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
146	* context. */
147	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
148	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
149	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
150
151	/** Log the VM-exit reason with an easily visible marker to identify it in a
152	* potential sea of logging data. */
153	#define HMVMX_LOG_EXIT(a_pVCpu, a_uExitReason) \
154	do { \
155	Log4(("VM-exit: vcpu[%RU32] %85s -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (a_pVCpu)->idCpu, \
156	HMGetVmxExitName(a_uExitReason))); \
157	} while (0) \
158
159
160	/*********************************************************************************************************************************
161	* Structures and Typedefs *
162	*********************************************************************************************************************************/
163	/**
164	* Memory operand read or write access.
165	*/
166	typedef enum VMXMEMACCESS
167	{
168	VMXMEMACCESS_READ = 0,
169	VMXMEMACCESS_WRITE = 1
170	} VMXMEMACCESS;
171
172
173	/**
174	* VMX VM-exit handler.
175	*
176	* @returns Strict VBox status code (i.e. informational status codes too).
177	* @param pVCpu The cross context virtual CPU structure.
178	* @param pVmxTransient The VMX-transient structure.
179	*/
180	#ifndef HMVMX_USE_FUNCTION_TABLE
181	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
182	#else
183	typedef DECLCALLBACKTYPE(VBOXSTRICTRC, FNVMXEXITHANDLER,(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient));
184	/** Pointer to VM-exit handler. */
185	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
186	#endif
187
188	/**
189	* VMX VM-exit handler, non-strict status code.
190	*
191	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
192	*
193	* @returns VBox status code, no informational status code returned.
194	* @param pVCpu The cross context virtual CPU structure.
195	* @param pVmxTransient The VMX-transient structure.
196	*
197	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
198	* use of that status code will be replaced with VINF_EM_SOMETHING
199	* later when switching over to IEM.
200	*/
201	#ifndef HMVMX_USE_FUNCTION_TABLE
202	typedef int FNVMXEXITHANDLERNSRC(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
203	#else
204	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
205	#endif
206
207
208	/*********************************************************************************************************************************
209	* Internal Functions *
210	*********************************************************************************************************************************/
211	#ifndef HMVMX_USE_FUNCTION_TABLE
212	DECLINLINE(VBOXSTRICTRC) vmxHCHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
213	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
214	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
215	#else
216	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
217	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
218	#endif
219	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
220	DECLINLINE(VBOXSTRICTRC) vmxHCHandleExitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
221	#endif
222
223	static int vmxHCImportGuestState(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
224
225	/** @name VM-exit handler prototypes.
226	* @{
227	*/
228	static FNVMXEXITHANDLER vmxHCExitXcptOrNmi;
229	static FNVMXEXITHANDLER vmxHCExitExtInt;
230	static FNVMXEXITHANDLER vmxHCExitTripleFault;
231	static FNVMXEXITHANDLERNSRC vmxHCExitIntWindow;
232	static FNVMXEXITHANDLERNSRC vmxHCExitNmiWindow;
233	static FNVMXEXITHANDLER vmxHCExitTaskSwitch;
234	static FNVMXEXITHANDLER vmxHCExitCpuid;
235	static FNVMXEXITHANDLER vmxHCExitGetsec;
236	static FNVMXEXITHANDLER vmxHCExitHlt;
237	static FNVMXEXITHANDLERNSRC vmxHCExitInvd;
238	static FNVMXEXITHANDLER vmxHCExitInvlpg;
239	static FNVMXEXITHANDLER vmxHCExitRdpmc;
240	static FNVMXEXITHANDLER vmxHCExitVmcall;
241	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
242	static FNVMXEXITHANDLER vmxHCExitVmclear;
243	static FNVMXEXITHANDLER vmxHCExitVmlaunch;
244	static FNVMXEXITHANDLER vmxHCExitVmptrld;
245	static FNVMXEXITHANDLER vmxHCExitVmptrst;
246	static FNVMXEXITHANDLER vmxHCExitVmread;
247	static FNVMXEXITHANDLER vmxHCExitVmresume;
248	static FNVMXEXITHANDLER vmxHCExitVmwrite;
249	static FNVMXEXITHANDLER vmxHCExitVmxoff;
250	static FNVMXEXITHANDLER vmxHCExitVmxon;
251	static FNVMXEXITHANDLER vmxHCExitInvvpid;
252	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
253	static FNVMXEXITHANDLER vmxHCExitInvept;
254	# endif
255	#endif
256	static FNVMXEXITHANDLER vmxHCExitRdtsc;
257	static FNVMXEXITHANDLER vmxHCExitMovCRx;
258	static FNVMXEXITHANDLER vmxHCExitMovDRx;
259	static FNVMXEXITHANDLER vmxHCExitIoInstr;
260	static FNVMXEXITHANDLER vmxHCExitRdmsr;
261	static FNVMXEXITHANDLER vmxHCExitWrmsr;
262	static FNVMXEXITHANDLER vmxHCExitMwait;
263	static FNVMXEXITHANDLER vmxHCExitMtf;
264	static FNVMXEXITHANDLER vmxHCExitMonitor;
265	static FNVMXEXITHANDLER vmxHCExitPause;
266	static FNVMXEXITHANDLERNSRC vmxHCExitTprBelowThreshold;
267	static FNVMXEXITHANDLER vmxHCExitApicAccess;
268	static FNVMXEXITHANDLER vmxHCExitEptViolation;
269	static FNVMXEXITHANDLER vmxHCExitEptMisconfig;
270	static FNVMXEXITHANDLER vmxHCExitRdtscp;
271	static FNVMXEXITHANDLER vmxHCExitPreemptTimer;
272	static FNVMXEXITHANDLERNSRC vmxHCExitWbinvd;
273	static FNVMXEXITHANDLER vmxHCExitXsetbv;
274	static FNVMXEXITHANDLER vmxHCExitInvpcid;
275	static FNVMXEXITHANDLERNSRC vmxHCExitSetPendingXcptUD;
276	static FNVMXEXITHANDLERNSRC vmxHCExitErrInvalidGuestState;
277	static FNVMXEXITHANDLERNSRC vmxHCExitErrUnexpected;
278	/** @} */
279
280	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
281	/** @name Nested-guest VM-exit handler prototypes.
282	* @{
283	*/
284	static FNVMXEXITHANDLER vmxHCExitXcptOrNmiNested;
285	static FNVMXEXITHANDLER vmxHCExitTripleFaultNested;
286	static FNVMXEXITHANDLERNSRC vmxHCExitIntWindowNested;
287	static FNVMXEXITHANDLERNSRC vmxHCExitNmiWindowNested;
288	static FNVMXEXITHANDLER vmxHCExitTaskSwitchNested;
289	static FNVMXEXITHANDLER vmxHCExitHltNested;
290	static FNVMXEXITHANDLER vmxHCExitInvlpgNested;
291	static FNVMXEXITHANDLER vmxHCExitRdpmcNested;
292	static FNVMXEXITHANDLER vmxHCExitVmreadVmwriteNested;
293	static FNVMXEXITHANDLER vmxHCExitRdtscNested;
294	static FNVMXEXITHANDLER vmxHCExitMovCRxNested;
295	static FNVMXEXITHANDLER vmxHCExitMovDRxNested;
296	static FNVMXEXITHANDLER vmxHCExitIoInstrNested;
297	static FNVMXEXITHANDLER vmxHCExitRdmsrNested;
298	static FNVMXEXITHANDLER vmxHCExitWrmsrNested;
299	static FNVMXEXITHANDLER vmxHCExitMwaitNested;
300	static FNVMXEXITHANDLER vmxHCExitMtfNested;
301	static FNVMXEXITHANDLER vmxHCExitMonitorNested;
302	static FNVMXEXITHANDLER vmxHCExitPauseNested;
303	static FNVMXEXITHANDLERNSRC vmxHCExitTprBelowThresholdNested;
304	static FNVMXEXITHANDLER vmxHCExitApicAccessNested;
305	static FNVMXEXITHANDLER vmxHCExitApicWriteNested;
306	static FNVMXEXITHANDLER vmxHCExitVirtEoiNested;
307	static FNVMXEXITHANDLER vmxHCExitRdtscpNested;
308	static FNVMXEXITHANDLERNSRC vmxHCExitWbinvdNested;
309	static FNVMXEXITHANDLER vmxHCExitInvpcidNested;
310	static FNVMXEXITHANDLERNSRC vmxHCExitErrInvalidGuestStateNested;
311	static FNVMXEXITHANDLER vmxHCExitInstrNested;
312	static FNVMXEXITHANDLER vmxHCExitInstrWithInfoNested;
313	/** @} */
314	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
315
316
317	/*********************************************************************************************************************************
318	* Global Variables *
319	*********************************************************************************************************************************/
320	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
321	/**
322	* Array of all VMCS fields.
323	* Any fields added to the VT-x spec. should be added here.
324	*
325	* Currently only used to derive shadow VMCS fields for hardware-assisted execution
326	* of nested-guests.
327	*/
328	static const uint32_t g_aVmcsFields[] =
329	{
330	/* 16-bit control fields. */
331	VMX_VMCS16_VPID,
332	VMX_VMCS16_POSTED_INT_NOTIFY_VECTOR,
333	VMX_VMCS16_EPTP_INDEX,
334
335	/* 16-bit guest-state fields. */
336	VMX_VMCS16_GUEST_ES_SEL,
337	VMX_VMCS16_GUEST_CS_SEL,
338	VMX_VMCS16_GUEST_SS_SEL,
339	VMX_VMCS16_GUEST_DS_SEL,
340	VMX_VMCS16_GUEST_FS_SEL,
341	VMX_VMCS16_GUEST_GS_SEL,
342	VMX_VMCS16_GUEST_LDTR_SEL,
343	VMX_VMCS16_GUEST_TR_SEL,
344	VMX_VMCS16_GUEST_INTR_STATUS,
345	VMX_VMCS16_GUEST_PML_INDEX,
346
347	/* 16-bits host-state fields. */
348	VMX_VMCS16_HOST_ES_SEL,
349	VMX_VMCS16_HOST_CS_SEL,
350	VMX_VMCS16_HOST_SS_SEL,
351	VMX_VMCS16_HOST_DS_SEL,
352	VMX_VMCS16_HOST_FS_SEL,
353	VMX_VMCS16_HOST_GS_SEL,
354	VMX_VMCS16_HOST_TR_SEL,
355
356	/* 64-bit control fields. */
357	VMX_VMCS64_CTRL_IO_BITMAP_A_FULL,
358	VMX_VMCS64_CTRL_IO_BITMAP_A_HIGH,
359	VMX_VMCS64_CTRL_IO_BITMAP_B_FULL,
360	VMX_VMCS64_CTRL_IO_BITMAP_B_HIGH,
361	VMX_VMCS64_CTRL_MSR_BITMAP_FULL,
362	VMX_VMCS64_CTRL_MSR_BITMAP_HIGH,
363	VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL,
364	VMX_VMCS64_CTRL_EXIT_MSR_STORE_HIGH,
365	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL,
366	VMX_VMCS64_CTRL_EXIT_MSR_LOAD_HIGH,
367	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL,
368	VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_HIGH,
369	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL,
370	VMX_VMCS64_CTRL_EXEC_VMCS_PTR_HIGH,
371	VMX_VMCS64_CTRL_EXEC_PML_ADDR_FULL,
372	VMX_VMCS64_CTRL_EXEC_PML_ADDR_HIGH,
373	VMX_VMCS64_CTRL_TSC_OFFSET_FULL,
374	VMX_VMCS64_CTRL_TSC_OFFSET_HIGH,
375	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL,
376	VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_HIGH,
377	VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL,
378	VMX_VMCS64_CTRL_APIC_ACCESSADDR_HIGH,
379	VMX_VMCS64_CTRL_POSTED_INTR_DESC_FULL,
380	VMX_VMCS64_CTRL_POSTED_INTR_DESC_HIGH,
381	VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL,
382	VMX_VMCS64_CTRL_VMFUNC_CTRLS_HIGH,
383	VMX_VMCS64_CTRL_EPTP_FULL,
384	VMX_VMCS64_CTRL_EPTP_HIGH,
385	VMX_VMCS64_CTRL_EOI_BITMAP_0_FULL,
386	VMX_VMCS64_CTRL_EOI_BITMAP_0_HIGH,
387	VMX_VMCS64_CTRL_EOI_BITMAP_1_FULL,
388	VMX_VMCS64_CTRL_EOI_BITMAP_1_HIGH,
389	VMX_VMCS64_CTRL_EOI_BITMAP_2_FULL,
390	VMX_VMCS64_CTRL_EOI_BITMAP_2_HIGH,
391	VMX_VMCS64_CTRL_EOI_BITMAP_3_FULL,
392	VMX_VMCS64_CTRL_EOI_BITMAP_3_HIGH,
393	VMX_VMCS64_CTRL_EPTP_LIST_FULL,
394	VMX_VMCS64_CTRL_EPTP_LIST_HIGH,
395	VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL,
396	VMX_VMCS64_CTRL_VMREAD_BITMAP_HIGH,
397	VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL,
398	VMX_VMCS64_CTRL_VMWRITE_BITMAP_HIGH,
399	VMX_VMCS64_CTRL_VE_XCPT_INFO_ADDR_FULL,
400	VMX_VMCS64_CTRL_VE_XCPT_INFO_ADDR_HIGH,
401	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_FULL,
402	VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_HIGH,
403	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_FULL,
404	VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_HIGH,
405	VMX_VMCS64_CTRL_SPPTP_FULL,
406	VMX_VMCS64_CTRL_SPPTP_HIGH,
407	VMX_VMCS64_CTRL_TSC_MULTIPLIER_FULL,
408	VMX_VMCS64_CTRL_TSC_MULTIPLIER_HIGH,
409	VMX_VMCS64_CTRL_PROC_EXEC3_FULL,
410	VMX_VMCS64_CTRL_PROC_EXEC3_HIGH,
411	VMX_VMCS64_CTRL_ENCLV_EXITING_BITMAP_FULL,
412	VMX_VMCS64_CTRL_ENCLV_EXITING_BITMAP_HIGH,
413
414	/* 64-bit read-only data fields. */
415	VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL,
416	VMX_VMCS64_RO_GUEST_PHYS_ADDR_HIGH,
417
418	/* 64-bit guest-state fields. */
419	VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL,
420	VMX_VMCS64_GUEST_VMCS_LINK_PTR_HIGH,
421	VMX_VMCS64_GUEST_DEBUGCTL_FULL,
422	VMX_VMCS64_GUEST_DEBUGCTL_HIGH,
423	VMX_VMCS64_GUEST_PAT_FULL,
424	VMX_VMCS64_GUEST_PAT_HIGH,
425	VMX_VMCS64_GUEST_EFER_FULL,
426	VMX_VMCS64_GUEST_EFER_HIGH,
427	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL,
428	VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_HIGH,
429	VMX_VMCS64_GUEST_PDPTE0_FULL,
430	VMX_VMCS64_GUEST_PDPTE0_HIGH,
431	VMX_VMCS64_GUEST_PDPTE1_FULL,
432	VMX_VMCS64_GUEST_PDPTE1_HIGH,
433	VMX_VMCS64_GUEST_PDPTE2_FULL,
434	VMX_VMCS64_GUEST_PDPTE2_HIGH,
435	VMX_VMCS64_GUEST_PDPTE3_FULL,
436	VMX_VMCS64_GUEST_PDPTE3_HIGH,
437	VMX_VMCS64_GUEST_BNDCFGS_FULL,
438	VMX_VMCS64_GUEST_BNDCFGS_HIGH,
439	VMX_VMCS64_GUEST_RTIT_CTL_FULL,
440	VMX_VMCS64_GUEST_RTIT_CTL_HIGH,
441	VMX_VMCS64_GUEST_PKRS_FULL,
442	VMX_VMCS64_GUEST_PKRS_HIGH,
443
444	/* 64-bit host-state fields. */
445	VMX_VMCS64_HOST_PAT_FULL,
446	VMX_VMCS64_HOST_PAT_HIGH,
447	VMX_VMCS64_HOST_EFER_FULL,
448	VMX_VMCS64_HOST_EFER_HIGH,
449	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL,
450	VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_HIGH,
451	VMX_VMCS64_HOST_PKRS_FULL,
452	VMX_VMCS64_HOST_PKRS_HIGH,
453
454	/* 32-bit control fields. */
455	VMX_VMCS32_CTRL_PIN_EXEC,
456	VMX_VMCS32_CTRL_PROC_EXEC,
457	VMX_VMCS32_CTRL_EXCEPTION_BITMAP,
458	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK,
459	VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH,
460	VMX_VMCS32_CTRL_CR3_TARGET_COUNT,
461	VMX_VMCS32_CTRL_EXIT,
462	VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT,
463	VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT,
464	VMX_VMCS32_CTRL_ENTRY,
465	VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT,
466	VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO,
467	VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE,
468	VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH,
469	VMX_VMCS32_CTRL_TPR_THRESHOLD,
470	VMX_VMCS32_CTRL_PROC_EXEC2,
471	VMX_VMCS32_CTRL_PLE_GAP,
472	VMX_VMCS32_CTRL_PLE_WINDOW,
473
474	/* 32-bits read-only fields. */
475	VMX_VMCS32_RO_VM_INSTR_ERROR,
476	VMX_VMCS32_RO_EXIT_REASON,
477	VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO,
478	VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE,
479	VMX_VMCS32_RO_IDT_VECTORING_INFO,
480	VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE,
481	VMX_VMCS32_RO_EXIT_INSTR_LENGTH,
482	VMX_VMCS32_RO_EXIT_INSTR_INFO,
483
484	/* 32-bit guest-state fields. */
485	VMX_VMCS32_GUEST_ES_LIMIT,
486	VMX_VMCS32_GUEST_CS_LIMIT,
487	VMX_VMCS32_GUEST_SS_LIMIT,
488	VMX_VMCS32_GUEST_DS_LIMIT,
489	VMX_VMCS32_GUEST_FS_LIMIT,
490	VMX_VMCS32_GUEST_GS_LIMIT,
491	VMX_VMCS32_GUEST_LDTR_LIMIT,
492	VMX_VMCS32_GUEST_TR_LIMIT,
493	VMX_VMCS32_GUEST_GDTR_LIMIT,
494	VMX_VMCS32_GUEST_IDTR_LIMIT,
495	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
496	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
497	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
498	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
499	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
500	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS,
501	VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS,
502	VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS,
503	VMX_VMCS32_GUEST_INT_STATE,
504	VMX_VMCS32_GUEST_ACTIVITY_STATE,
505	VMX_VMCS32_GUEST_SMBASE,
506	VMX_VMCS32_GUEST_SYSENTER_CS,
507	VMX_VMCS32_PREEMPT_TIMER_VALUE,
508
509	/* 32-bit host-state fields. */
510	VMX_VMCS32_HOST_SYSENTER_CS,
511
512	/* Natural-width control fields. */
513	VMX_VMCS_CTRL_CR0_MASK,
514	VMX_VMCS_CTRL_CR4_MASK,
515	VMX_VMCS_CTRL_CR0_READ_SHADOW,
516	VMX_VMCS_CTRL_CR4_READ_SHADOW,
517	VMX_VMCS_CTRL_CR3_TARGET_VAL0,
518	VMX_VMCS_CTRL_CR3_TARGET_VAL1,
519	VMX_VMCS_CTRL_CR3_TARGET_VAL2,
520	VMX_VMCS_CTRL_CR3_TARGET_VAL3,
521
522	/* Natural-width read-only data fields. */
523	VMX_VMCS_RO_EXIT_QUALIFICATION,
524	VMX_VMCS_RO_IO_RCX,
525	VMX_VMCS_RO_IO_RSI,
526	VMX_VMCS_RO_IO_RDI,
527	VMX_VMCS_RO_IO_RIP,
528	VMX_VMCS_RO_GUEST_LINEAR_ADDR,
529
530	/* Natural-width guest-state field */
531	VMX_VMCS_GUEST_CR0,
532	VMX_VMCS_GUEST_CR3,
533	VMX_VMCS_GUEST_CR4,
534	VMX_VMCS_GUEST_ES_BASE,
535	VMX_VMCS_GUEST_CS_BASE,
536	VMX_VMCS_GUEST_SS_BASE,
537	VMX_VMCS_GUEST_DS_BASE,
538	VMX_VMCS_GUEST_FS_BASE,
539	VMX_VMCS_GUEST_GS_BASE,
540	VMX_VMCS_GUEST_LDTR_BASE,
541	VMX_VMCS_GUEST_TR_BASE,
542	VMX_VMCS_GUEST_GDTR_BASE,
543	VMX_VMCS_GUEST_IDTR_BASE,
544	VMX_VMCS_GUEST_DR7,
545	VMX_VMCS_GUEST_RSP,
546	VMX_VMCS_GUEST_RIP,
547	VMX_VMCS_GUEST_RFLAGS,
548	VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,
549	VMX_VMCS_GUEST_SYSENTER_ESP,
550	VMX_VMCS_GUEST_SYSENTER_EIP,
551	VMX_VMCS_GUEST_S_CET,
552	VMX_VMCS_GUEST_SSP,
553	VMX_VMCS_GUEST_INTR_SSP_TABLE_ADDR,
554
555	/* Natural-width host-state fields */
556	VMX_VMCS_HOST_CR0,
557	VMX_VMCS_HOST_CR3,
558	VMX_VMCS_HOST_CR4,
559	VMX_VMCS_HOST_FS_BASE,
560	VMX_VMCS_HOST_GS_BASE,
561	VMX_VMCS_HOST_TR_BASE,
562	VMX_VMCS_HOST_GDTR_BASE,
563	VMX_VMCS_HOST_IDTR_BASE,
564	VMX_VMCS_HOST_SYSENTER_ESP,
565	VMX_VMCS_HOST_SYSENTER_EIP,
566	VMX_VMCS_HOST_RSP,
567	VMX_VMCS_HOST_RIP,
568	VMX_VMCS_HOST_S_CET,
569	VMX_VMCS_HOST_SSP,
570	VMX_VMCS_HOST_INTR_SSP_TABLE_ADDR
571	};
572	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
573
574	#ifdef VBOX_STRICT
575	static const uint32_t g_aVmcsSegBase[] =
576	{
577	VMX_VMCS_GUEST_ES_BASE,
578	VMX_VMCS_GUEST_CS_BASE,
579	VMX_VMCS_GUEST_SS_BASE,
580	VMX_VMCS_GUEST_DS_BASE,
581	VMX_VMCS_GUEST_FS_BASE,
582	VMX_VMCS_GUEST_GS_BASE
583	};
584	static const uint32_t g_aVmcsSegSel[] =
585	{
586	VMX_VMCS16_GUEST_ES_SEL,
587	VMX_VMCS16_GUEST_CS_SEL,
588	VMX_VMCS16_GUEST_SS_SEL,
589	VMX_VMCS16_GUEST_DS_SEL,
590	VMX_VMCS16_GUEST_FS_SEL,
591	VMX_VMCS16_GUEST_GS_SEL
592	};
593	static const uint32_t g_aVmcsSegLimit[] =
594	{
595	VMX_VMCS32_GUEST_ES_LIMIT,
596	VMX_VMCS32_GUEST_CS_LIMIT,
597	VMX_VMCS32_GUEST_SS_LIMIT,
598	VMX_VMCS32_GUEST_DS_LIMIT,
599	VMX_VMCS32_GUEST_FS_LIMIT,
600	VMX_VMCS32_GUEST_GS_LIMIT
601	};
602	static const uint32_t g_aVmcsSegAttr[] =
603	{
604	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
605	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
606	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
607	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
608	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
609	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
610	};
611	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
612	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
613	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
614	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
615	#endif /* VBOX_STRICT */
616
617	#ifdef HMVMX_USE_FUNCTION_TABLE
618	/**
619	* VMX_EXIT dispatch table.
620	*/
621	static const struct CLANG11NOTHROWWEIRDNESS { PFNVMXEXITHANDLER pfn; } g_aVMExitHandlers[VMX_EXIT_MAX + 1] =
622	{
623	/* 0 VMX_EXIT_XCPT_OR_NMI */ { vmxHCExitXcptOrNmi },
624	/* 1 VMX_EXIT_EXT_INT */ { vmxHCExitExtInt },
625	/* 2 VMX_EXIT_TRIPLE_FAULT */ { vmxHCExitTripleFault },
626	/* 3 VMX_EXIT_INIT_SIGNAL */ { vmxHCExitErrUnexpected },
627	/* 4 VMX_EXIT_SIPI */ { vmxHCExitErrUnexpected },
628	/* 5 VMX_EXIT_IO_SMI */ { vmxHCExitErrUnexpected },
629	/* 6 VMX_EXIT_SMI */ { vmxHCExitErrUnexpected },
630	/* 7 VMX_EXIT_INT_WINDOW */ { vmxHCExitIntWindow },
631	/* 8 VMX_EXIT_NMI_WINDOW */ { vmxHCExitNmiWindow },
632	/* 9 VMX_EXIT_TASK_SWITCH */ { vmxHCExitTaskSwitch },
633	/* 10 VMX_EXIT_CPUID */ { vmxHCExitCpuid },
634	/* 11 VMX_EXIT_GETSEC */ { vmxHCExitGetsec },
635	/* 12 VMX_EXIT_HLT */ { vmxHCExitHlt },
636	/* 13 VMX_EXIT_INVD */ { vmxHCExitInvd },
637	/* 14 VMX_EXIT_INVLPG */ { vmxHCExitInvlpg },
638	/* 15 VMX_EXIT_RDPMC */ { vmxHCExitRdpmc },
639	/* 16 VMX_EXIT_RDTSC */ { vmxHCExitRdtsc },
640	/* 17 VMX_EXIT_RSM */ { vmxHCExitErrUnexpected },
641	/* 18 VMX_EXIT_VMCALL */ { vmxHCExitVmcall },
642	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
643	/* 19 VMX_EXIT_VMCLEAR */ { vmxHCExitVmclear },
644	/* 20 VMX_EXIT_VMLAUNCH */ { vmxHCExitVmlaunch },
645	/* 21 VMX_EXIT_VMPTRLD */ { vmxHCExitVmptrld },
646	/* 22 VMX_EXIT_VMPTRST */ { vmxHCExitVmptrst },
647	/* 23 VMX_EXIT_VMREAD */ { vmxHCExitVmread },
648	/* 24 VMX_EXIT_VMRESUME */ { vmxHCExitVmresume },
649	/* 25 VMX_EXIT_VMWRITE */ { vmxHCExitVmwrite },
650	/* 26 VMX_EXIT_VMXOFF */ { vmxHCExitVmxoff },
651	/* 27 VMX_EXIT_VMXON */ { vmxHCExitVmxon },
652	#else
653	/* 19 VMX_EXIT_VMCLEAR */ { vmxHCExitSetPendingXcptUD },
654	/* 20 VMX_EXIT_VMLAUNCH */ { vmxHCExitSetPendingXcptUD },
655	/* 21 VMX_EXIT_VMPTRLD */ { vmxHCExitSetPendingXcptUD },
656	/* 22 VMX_EXIT_VMPTRST */ { vmxHCExitSetPendingXcptUD },
657	/* 23 VMX_EXIT_VMREAD */ { vmxHCExitSetPendingXcptUD },
658	/* 24 VMX_EXIT_VMRESUME */ { vmxHCExitSetPendingXcptUD },
659	/* 25 VMX_EXIT_VMWRITE */ { vmxHCExitSetPendingXcptUD },
660	/* 26 VMX_EXIT_VMXOFF */ { vmxHCExitSetPendingXcptUD },
661	/* 27 VMX_EXIT_VMXON */ { vmxHCExitSetPendingXcptUD },
662	#endif
663	/* 28 VMX_EXIT_MOV_CRX */ { vmxHCExitMovCRx },
664	/* 29 VMX_EXIT_MOV_DRX */ { vmxHCExitMovDRx },
665	/* 30 VMX_EXIT_IO_INSTR */ { vmxHCExitIoInstr },
666	/* 31 VMX_EXIT_RDMSR */ { vmxHCExitRdmsr },
667	/* 32 VMX_EXIT_WRMSR */ { vmxHCExitWrmsr },
668	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ { vmxHCExitErrInvalidGuestState },
669	/* 34 VMX_EXIT_ERR_MSR_LOAD */ { vmxHCExitErrUnexpected },
670	/* 35 UNDEFINED */ { vmxHCExitErrUnexpected },
671	/* 36 VMX_EXIT_MWAIT */ { vmxHCExitMwait },
672	/* 37 VMX_EXIT_MTF */ { vmxHCExitMtf },
673	/* 38 UNDEFINED */ { vmxHCExitErrUnexpected },
674	/* 39 VMX_EXIT_MONITOR */ { vmxHCExitMonitor },
675	/* 40 VMX_EXIT_PAUSE */ { vmxHCExitPause },
676	/* 41 VMX_EXIT_ERR_MACHINE_CHECK */ { vmxHCExitErrUnexpected },
677	/* 42 UNDEFINED */ { vmxHCExitErrUnexpected },
678	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ { vmxHCExitTprBelowThreshold },
679	/* 44 VMX_EXIT_APIC_ACCESS */ { vmxHCExitApicAccess },
680	/* 45 VMX_EXIT_VIRTUALIZED_EOI */ { vmxHCExitErrUnexpected },
681	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ { vmxHCExitErrUnexpected },
682	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ { vmxHCExitErrUnexpected },
683	/* 48 VMX_EXIT_EPT_VIOLATION */ { vmxHCExitEptViolation },
684	/* 49 VMX_EXIT_EPT_MISCONFIG */ { vmxHCExitEptMisconfig },
685	#if defined(VBOX_WITH_NESTED_HWVIRT_VMX) && defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT)
686	/* 50 VMX_EXIT_INVEPT */ { vmxHCExitInvept },
687	#else
688	/* 50 VMX_EXIT_INVEPT */ { vmxHCExitSetPendingXcptUD },
689	#endif
690	/* 51 VMX_EXIT_RDTSCP */ { vmxHCExitRdtscp },
691	/* 52 VMX_EXIT_PREEMPT_TIMER */ { vmxHCExitPreemptTimer },
692	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
693	/* 53 VMX_EXIT_INVVPID */ { vmxHCExitInvvpid },
694	#else
695	/* 53 VMX_EXIT_INVVPID */ { vmxHCExitSetPendingXcptUD },
696	#endif
697	/* 54 VMX_EXIT_WBINVD */ { vmxHCExitWbinvd },
698	/* 55 VMX_EXIT_XSETBV */ { vmxHCExitXsetbv },
699	/* 56 VMX_EXIT_APIC_WRITE */ { vmxHCExitErrUnexpected },
700	/* 57 VMX_EXIT_RDRAND */ { vmxHCExitErrUnexpected },
701	/* 58 VMX_EXIT_INVPCID */ { vmxHCExitInvpcid },
702	/* 59 VMX_EXIT_VMFUNC */ { vmxHCExitErrUnexpected },
703	/* 60 VMX_EXIT_ENCLS */ { vmxHCExitErrUnexpected },
704	/* 61 VMX_EXIT_RDSEED */ { vmxHCExitErrUnexpected },
705	/* 62 VMX_EXIT_PML_FULL */ { vmxHCExitErrUnexpected },
706	/* 63 VMX_EXIT_XSAVES */ { vmxHCExitErrUnexpected },
707	/* 64 VMX_EXIT_XRSTORS */ { vmxHCExitErrUnexpected },
708	/* 65 UNDEFINED */ { vmxHCExitErrUnexpected },
709	/* 66 VMX_EXIT_SPP_EVENT */ { vmxHCExitErrUnexpected },
710	/* 67 VMX_EXIT_UMWAIT */ { vmxHCExitErrUnexpected },
711	/* 68 VMX_EXIT_TPAUSE */ { vmxHCExitErrUnexpected },
712	/* 69 VMX_EXIT_LOADIWKEY */ { vmxHCExitErrUnexpected },
713	};
714	#endif /* HMVMX_USE_FUNCTION_TABLE */
715
716	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
717	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
718	{
719	/* 0 */ "(Not Used)",
720	/* 1 */ "VMCALL executed in VMX root operation.",
721	/* 2 */ "VMCLEAR with invalid physical address.",
722	/* 3 */ "VMCLEAR with VMXON pointer.",
723	/* 4 */ "VMLAUNCH with non-clear VMCS.",
724	/* 5 */ "VMRESUME with non-launched VMCS.",
725	/* 6 */ "VMRESUME after VMXOFF",
726	/* 7 */ "VM-entry with invalid control fields.",
727	/* 8 */ "VM-entry with invalid host state fields.",
728	/* 9 */ "VMPTRLD with invalid physical address.",
729	/* 10 */ "VMPTRLD with VMXON pointer.",
730	/* 11 */ "VMPTRLD with incorrect revision identifier.",
731	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
732	/* 13 */ "VMWRITE to read-only VMCS component.",
733	/* 14 */ "(Not Used)",
734	/* 15 */ "VMXON executed in VMX root operation.",
735	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
736	/* 17 */ "VM-entry with non-launched executing VMCS.",
737	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
738	/* 19 */ "VMCALL with non-clear VMCS.",
739	/* 20 */ "VMCALL with invalid VM-exit control fields.",
740	/* 21 */ "(Not Used)",
741	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
742	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
743	/* 24 */ "VMCALL with invalid SMM-monitor features.",
744	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
745	/* 26 */ "VM-entry with events blocked by MOV SS.",
746	/* 27 */ "(Not Used)",
747	/* 28 */ "Invalid operand to INVEPT/INVVPID."
748	};
749	#endif /* VBOX_STRICT && LOG_ENABLED */
750
751
752	/**
753	* Gets the CR0 guest/host mask.
754	*
755	* These bits typically does not change through the lifetime of a VM. Any bit set in
756	* this mask is owned by the host/hypervisor and would cause a VM-exit when modified
757	* by the guest.
758	*
759	* @returns The CR0 guest/host mask.
760	* @param pVCpu The cross context virtual CPU structure.
761	*/
762	static uint64_t vmxHCGetFixedCr0Mask(PCVMCPUCC pVCpu)
763	{
764	/*
765	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
766	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
767	*
768	* Furthermore, modifications to any bits that are reserved/unspecified currently
769	* by the Intel spec. must also cause a VM-exit. This prevents unpredictable behavior
770	* when future CPUs specify and use currently reserved/unspecified bits.
771	*/
772	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
773	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
774	* and @bugref{6944}. */
775	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
776	return ( X86_CR0_PE
777	\| X86_CR0_NE
778	\| (VM_IS_VMX_NESTED_PAGING(pVM) ? 0 : X86_CR0_WP)
779	\| X86_CR0_PG
780	\| VMX_EXIT_HOST_CR0_IGNORE_MASK);
781	}
782
783
784	/**
785	* Gets the CR4 guest/host mask.
786	*
787	* These bits typically does not change through the lifetime of a VM. Any bit set in
788	* this mask is owned by the host/hypervisor and would cause a VM-exit when modified
789	* by the guest.
790	*
791	* @returns The CR4 guest/host mask.
792	* @param pVCpu The cross context virtual CPU structure.
793	*/
794	static uint64_t vmxHCGetFixedCr4Mask(PCVMCPUCC pVCpu)
795	{
796	/*
797	* We construct a mask of all CR4 bits that the guest can modify without causing
798	* a VM-exit. Then invert this mask to obtain all CR4 bits that should cause
799	* a VM-exit when the guest attempts to modify them when executing using
800	* hardware-assisted VMX.
801	*
802	* When a feature is not exposed to the guest (and may be present on the host),
803	* we want to intercept guest modifications to the bit so we can emulate proper
804	* behavior (e.g., #GP).
805	*
806	* Furthermore, only modifications to those bits that don't require immediate
807	* emulation is allowed. For e.g., PCIDE is excluded because the behavior
808	* depends on CR3 which might not always be the guest value while executing
809	* using hardware-assisted VMX.
810	*/
811	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
812	bool const fFsGsBase = pVM->cpum.ro.GuestFeatures.fFsGsBase;
813	bool const fXSaveRstor = pVM->cpum.ro.GuestFeatures.fXSaveRstor;
814	bool const fFxSaveRstor = pVM->cpum.ro.GuestFeatures.fFxSaveRstor;
815
816	/*
817	* Paranoia.
818	* Ensure features exposed to the guest are present on the host.
819	*/
820	Assert(!fFsGsBase \|\| pVM->cpum.ro.HostFeatures.fFsGsBase);
821	Assert(!fXSaveRstor \|\| pVM->cpum.ro.HostFeatures.fXSaveRstor);
822	Assert(!fFxSaveRstor \|\| pVM->cpum.ro.HostFeatures.fFxSaveRstor);
823
824	uint64_t const fGstMask = ( X86_CR4_PVI
825	\| X86_CR4_TSD
826	\| X86_CR4_DE
827	\| X86_CR4_MCE
828	\| X86_CR4_PCE
829	\| X86_CR4_OSXMMEEXCPT
830	\| (fFsGsBase ? X86_CR4_FSGSBASE : 0)
831	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
832	\| (fFxSaveRstor ? X86_CR4_OSFXSR : 0));
833	return ~fGstMask;
834	}
835
836
837	/**
838	* Adds one or more exceptions to the exception bitmap and commits it to the current
839	* VMCS.
840	*
841	* @param pVCpu The cross context virtual CPU structure.
842	* @param pVmxTransient The VMX-transient structure.
843	* @param uXcptMask The exception(s) to add.
844	*/
845	static void vmxHCAddXcptInterceptMask(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
846	{
847	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
848	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
849	if ((uXcptBitmap & uXcptMask) != uXcptMask)
850	{
851	uXcptBitmap \|= uXcptMask;
852	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
853	AssertRC(rc);
854	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
855	}
856	}
857
858
859	/**
860	* Adds an exception to the exception bitmap and commits it to the current VMCS.
861	*
862	* @param pVCpu The cross context virtual CPU structure.
863	* @param pVmxTransient The VMX-transient structure.
864	* @param uXcpt The exception to add.
865	*/
866	static void vmxHCAddXcptIntercept(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
867	{
868	Assert(uXcpt <= X86_XCPT_LAST);
869	vmxHCAddXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT_32(uXcpt));
870	}
871
872
873	/**
874	* Remove one or more exceptions from the exception bitmap and commits it to the
875	* current VMCS.
876	*
877	* This takes care of not removing the exception intercept if a nested-guest
878	* requires the exception to be intercepted.
879	*
880	* @returns VBox status code.
881	* @param pVCpu The cross context virtual CPU structure.
882	* @param pVmxTransient The VMX-transient structure.
883	* @param uXcptMask The exception(s) to remove.
884	*/
885	static int vmxHCRemoveXcptInterceptMask(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
886	{
887	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
888	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
889	if (u32XcptBitmap & uXcptMask)
890	{
891	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
892	if (!pVmxTransient->fIsNestedGuest)
893	{ /* likely */ }
894	else
895	uXcptMask &= ~pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32XcptBitmap;
896	#endif
897	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
898	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
899	\| RT_BIT(X86_XCPT_DE)
900	\| RT_BIT(X86_XCPT_NM)
901	\| RT_BIT(X86_XCPT_TS)
902	\| RT_BIT(X86_XCPT_UD)
903	\| RT_BIT(X86_XCPT_NP)
904	\| RT_BIT(X86_XCPT_SS)
905	\| RT_BIT(X86_XCPT_GP)
906	\| RT_BIT(X86_XCPT_PF)
907	\| RT_BIT(X86_XCPT_MF));
908	#elif defined(HMVMX_ALWAYS_TRAP_PF)
909	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
910	#endif
911	if (uXcptMask)
912	{
913	/* Validate we are not removing any essential exception intercepts. */
914	#ifndef IN_NEM_DARWIN
915	Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF)));
916	#else
917	Assert(!(uXcptMask & RT_BIT(X86_XCPT_PF)));
918	#endif
919	NOREF(pVCpu);
920	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
921	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
922
923	/* Remove it from the exception bitmap. */
924	u32XcptBitmap &= ~uXcptMask;
925
926	/* Commit and update the cache if necessary. */
927	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
928	{
929	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
930	AssertRC(rc);
931	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
932	}
933	}
934	}
935	return VINF_SUCCESS;
936	}
937
938
939	/**
940	* Remove an exceptions from the exception bitmap and commits it to the current
941	* VMCS.
942	*
943	* @returns VBox status code.
944	* @param pVCpu The cross context virtual CPU structure.
945	* @param pVmxTransient The VMX-transient structure.
946	* @param uXcpt The exception to remove.
947	*/
948	static int vmxHCRemoveXcptIntercept(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
949	{
950	return vmxHCRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
951	}
952
953
954	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
955	/**
956	* Loads the shadow VMCS specified by the VMCS info. object.
957	*
958	* @returns VBox status code.
959	* @param pVmcsInfo The VMCS info. object.
960	*
961	* @remarks Can be called with interrupts disabled.
962	*/
963	static int vmxHCLoadShadowVmcs(PVMXVMCSINFO pVmcsInfo)
964	{
965	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
966	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
967
968	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysShadowVmcs);
969	if (RT_SUCCESS(rc))
970	pVmcsInfo->fShadowVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
971	return rc;
972	}
973
974
975	/**
976	* Clears the shadow VMCS specified by the VMCS info. object.
977	*
978	* @returns VBox status code.
979	* @param pVmcsInfo The VMCS info. object.
980	*
981	* @remarks Can be called with interrupts disabled.
982	*/
983	static int vmxHCClearShadowVmcs(PVMXVMCSINFO pVmcsInfo)
984	{
985	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
986	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
987
988	int rc = VMXClearVmcs(pVmcsInfo->HCPhysShadowVmcs);
989	if (RT_SUCCESS(rc))
990	pVmcsInfo->fShadowVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
991	return rc;
992	}
993
994
995	/**
996	* Switches from and to the specified VMCSes.
997	*
998	* @returns VBox status code.
999	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
1000	* @param pVmcsInfoTo The VMCS info. object we are switching to.
1001	*
1002	* @remarks Called with interrupts disabled.
1003	*/
1004	static int vmxHCSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
1005	{
1006	/*
1007	* Clear the VMCS we are switching out if it has not already been cleared.
1008	* This will sync any CPU internal data back to the VMCS.
1009	*/
1010	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1011	{
1012	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
1013	if (RT_SUCCESS(rc))
1014	{
1015	/*
1016	* The shadow VMCS, if any, would not be active at this point since we
1017	* would have cleared it while importing the virtual hardware-virtualization
1018	* state as part the VMLAUNCH/VMRESUME VM-exit. Hence, there's no need to
1019	* clear the shadow VMCS here, just assert for safety.
1020	*/
1021	Assert(!pVmcsInfoFrom->pvShadowVmcs \|\| pVmcsInfoFrom->fShadowVmcsState == VMX_V_VMCS_LAUNCH_STATE_CLEAR);
1022	}
1023	else
1024	return rc;
1025	}
1026
1027	/*
1028	* Clear the VMCS we are switching to if it has not already been cleared.
1029	* This will initialize the VMCS launch state to "clear" required for loading it.
1030	*
1031	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
1032	*/
1033	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
1034	{
1035	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
1036	if (RT_SUCCESS(rc))
1037	{ /* likely */ }
1038	else
1039	return rc;
1040	}
1041
1042	/*
1043	* Finally, load the VMCS we are switching to.
1044	*/
1045	return hmR0VmxLoadVmcs(pVmcsInfoTo);
1046	}
1047
1048
1049	/**
1050	* Switches between the guest VMCS and the nested-guest VMCS as specified by the
1051	* caller.
1052	*
1053	* @returns VBox status code.
1054	* @param pVCpu The cross context virtual CPU structure.
1055	* @param fSwitchToNstGstVmcs Whether to switch to the nested-guest VMCS (pass
1056	* true) or guest VMCS (pass false).
1057	*/
1058	static int vmxHCSwitchToGstOrNstGstVmcs(PVMCPUCC pVCpu, bool fSwitchToNstGstVmcs)
1059	{
1060	/* Ensure we have synced everything from the guest-CPU context to the VMCS before switching. */
1061	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
1062
1063	PVMXVMCSINFO pVmcsInfoFrom;
1064	PVMXVMCSINFO pVmcsInfoTo;
1065	if (fSwitchToNstGstVmcs)
1066	{
1067	pVmcsInfoFrom = &pVCpu->hmr0.s.vmx.VmcsInfo;
1068	pVmcsInfoTo = &pVCpu->hmr0.s.vmx.VmcsInfoNstGst;
1069	}
1070	else
1071	{
1072	pVmcsInfoFrom = &pVCpu->hmr0.s.vmx.VmcsInfoNstGst;
1073	pVmcsInfoTo = &pVCpu->hmr0.s.vmx.VmcsInfo;
1074	}
1075
1076	/*
1077	* Disable interrupts to prevent being preempted while we switch the current VMCS as the
1078	* preemption hook code path acquires the current VMCS.
1079	*/
1080	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1081
1082	int rc = vmxHCSwitchVmcs(pVmcsInfoFrom, pVmcsInfoTo);
1083	if (RT_SUCCESS(rc))
1084	{
1085	pVCpu->hmr0.s.vmx.fSwitchedToNstGstVmcs = fSwitchToNstGstVmcs;
1086	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcsCopyForRing3 = fSwitchToNstGstVmcs;
1087
1088	/*
1089	* If we are switching to a VMCS that was executed on a different host CPU or was
1090	* never executed before, flag that we need to export the host state before executing
1091	* guest/nested-guest code using hardware-assisted VMX.
1092	*
1093	* This could probably be done in a preemptible context since the preemption hook
1094	* will flag the necessary change in host context. However, since preemption is
1095	* already disabled and to avoid making assumptions about host specific code in
1096	* RTMpCpuId when called with preemption enabled, we'll do this while preemption is
1097	* disabled.
1098	*/
1099	if (pVmcsInfoTo->idHostCpuState == RTMpCpuId())
1100	{ /* likely */ }
1101	else
1102	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE);
1103
1104	ASMSetFlags(fEFlags);
1105
1106	/*
1107	* We use a different VM-exit MSR-store areas for the guest and nested-guest. Hence,
1108	* flag that we need to update the host MSR values there. Even if we decide in the
1109	* future to share the VM-exit MSR-store area page between the guest and nested-guest,
1110	* if its content differs, we would have to update the host MSRs anyway.
1111	*/
1112	pVCpu->hmr0.s.vmx.fUpdatedHostAutoMsrs = false;
1113	}
1114	else
1115	ASMSetFlags(fEFlags);
1116	return rc;
1117	}
1118	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
1119
1120
1121	#ifdef VBOX_STRICT
1122	/**
1123	* Reads the VM-entry interruption-information field from the VMCS into the VMX
1124	* transient structure.
1125	*
1126	* @param pVCpu The cross context virtual CPU structure.
1127	* @param pVmxTransient The VMX-transient structure.
1128	*/
1129	DECLINLINE(void) vmxHCReadEntryIntInfoVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1130	{
1131	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
1132	AssertRC(rc);
1133	}
1134
1135
1136	/**
1137	* Reads the VM-entry exception error code field from the VMCS into
1138	* the VMX transient structure.
1139	*
1140	* @param pVCpu The cross context virtual CPU structure.
1141	* @param pVmxTransient The VMX-transient structure.
1142	*/
1143	DECLINLINE(void) vmxHCReadEntryXcptErrorCodeVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1144	{
1145	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
1146	AssertRC(rc);
1147	}
1148
1149
1150	/**
1151	* Reads the VM-entry exception error code field from the VMCS into
1152	* the VMX transient structure.
1153	*
1154	* @param pVCpu The cross context virtual CPU structure.
1155	* @param pVmxTransient The VMX-transient structure.
1156	*/
1157	DECLINLINE(void) vmxHCReadEntryInstrLenVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1158	{
1159	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
1160	AssertRC(rc);
1161	}
1162	#endif /* VBOX_STRICT */
1163
1164
1165	/**
1166	* Reads the VM-exit interruption-information field from the VMCS into the VMX
1167	* transient structure.
1168	*
1169	* @param pVCpu The cross context virtual CPU structure.
1170	* @param pVmxTransient The VMX-transient structure.
1171	*/
1172	DECLINLINE(void) vmxHCReadExitIntInfoVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1173	{
1174	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
1175	{
1176	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1177	AssertRC(rc);
1178	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
1179	}
1180	}
1181
1182
1183	/**
1184	* Reads the VM-exit interruption error code from the VMCS into the VMX
1185	* transient structure.
1186	*
1187	* @param pVCpu The cross context virtual CPU structure.
1188	* @param pVmxTransient The VMX-transient structure.
1189	*/
1190	DECLINLINE(void) vmxHCReadExitIntErrorCodeVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1191	{
1192	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
1193	{
1194	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1195	AssertRC(rc);
1196	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1197	}
1198	}
1199
1200
1201	/**
1202	* Reads the VM-exit instruction length field from the VMCS into the VMX
1203	* transient structure.
1204	*
1205	* @param pVCpu The cross context virtual CPU structure.
1206	* @param pVmxTransient The VMX-transient structure.
1207	*/
1208	DECLINLINE(void) vmxHCReadExitInstrLenVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1209	{
1210	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1211	{
1212	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbExitInstr);
1213	AssertRC(rc);
1214	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1215	}
1216	}
1217
1218
1219	/**
1220	* Reads the VM-exit instruction-information field from the VMCS into
1221	* the VMX transient structure.
1222	*
1223	* @param pVCpu The cross context virtual CPU structure.
1224	* @param pVmxTransient The VMX-transient structure.
1225	*/
1226	DECLINLINE(void) vmxHCReadExitInstrInfoVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1227	{
1228	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1229	{
1230	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1231	AssertRC(rc);
1232	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1233	}
1234	}
1235
1236
1237	/**
1238	* Reads the Exit Qualification from the VMCS into the VMX transient structure.
1239	*
1240	* @param pVCpu The cross context virtual CPU structure.
1241	* @param pVmxTransient The VMX-transient structure.
1242	*/
1243	DECLINLINE(void) vmxHCReadExitQualVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1244	{
1245	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1246	{
1247	int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual);
1248	AssertRC(rc);
1249	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1250	}
1251	}
1252
1253
1254	/**
1255	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1256	*
1257	* @param pVCpu The cross context virtual CPU structure.
1258	* @param pVmxTransient The VMX-transient structure.
1259	*/
1260	DECLINLINE(void) vmxHCReadGuestLinearAddrVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1261	{
1262	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1263	{
1264	int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr);
1265	AssertRC(rc);
1266	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1267	}
1268	}
1269
1270
1271	/**
1272	* Reads the Guest-physical address from the VMCS into the VMX transient structure.
1273	*
1274	* @param pVCpu The cross context virtual CPU structure.
1275	* @param pVmxTransient The VMX-transient structure.
1276	*/
1277	DECLINLINE(void) vmxHCReadGuestPhysicalAddrVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1278	{
1279	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_PHYSICAL_ADDR))
1280	{
1281	int rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr);
1282	AssertRC(rc);
1283	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_PHYSICAL_ADDR;
1284	}
1285	}
1286
1287	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1288	/**
1289	* Reads the Guest pending-debug exceptions from the VMCS into the VMX transient
1290	* structure.
1291	*
1292	* @param pVCpu The cross context virtual CPU structure.
1293	* @param pVmxTransient The VMX-transient structure.
1294	*/
1295	DECLINLINE(void) vmxHCReadGuestPendingDbgXctps(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1296	{
1297	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_PENDING_DBG_XCPTS))
1298	{
1299	int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &pVmxTransient->uGuestPendingDbgXcpts);
1300	AssertRC(rc);
1301	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_PENDING_DBG_XCPTS;
1302	}
1303	}
1304	#endif
1305
1306	/**
1307	* Reads the IDT-vectoring information field from the VMCS into the VMX
1308	* transient structure.
1309	*
1310	* @param pVCpu The cross context virtual CPU structure.
1311	* @param pVmxTransient The VMX-transient structure.
1312	*
1313	* @remarks No-long-jump zone!!!
1314	*/
1315	DECLINLINE(void) vmxHCReadIdtVectoringInfoVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1316	{
1317	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1318	{
1319	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1320	AssertRC(rc);
1321	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1322	}
1323	}
1324
1325
1326	/**
1327	* Reads the IDT-vectoring error code from the VMCS into the VMX
1328	* transient structure.
1329	*
1330	* @param pVCpu The cross context virtual CPU structure.
1331	* @param pVmxTransient The VMX-transient structure.
1332	*/
1333	DECLINLINE(void) vmxHCReadIdtVectoringErrorCodeVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1334	{
1335	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1336	{
1337	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1338	AssertRC(rc);
1339	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1340	}
1341	}
1342
1343	#ifdef HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
1344	/**
1345	* Reads all relevant read-only VMCS fields into the VMX transient structure.
1346	*
1347	* @param pVCpu The cross context virtual CPU structure.
1348	* @param pVmxTransient The VMX-transient structure.
1349	*/
1350	static void vmxHCReadAllRoFieldsVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1351	{
1352	int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual);
1353	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbExitInstr);
1354	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1355	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1356	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1357	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
1358	rc \|= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1359	rc \|= VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr);
1360	rc \|= VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr);
1361	AssertRC(rc);
1362	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION
1363	\| HMVMX_READ_EXIT_INSTR_LEN
1364	\| HMVMX_READ_EXIT_INSTR_INFO
1365	\| HMVMX_READ_IDT_VECTORING_INFO
1366	\| HMVMX_READ_IDT_VECTORING_ERROR_CODE
1367	\| HMVMX_READ_EXIT_INTERRUPTION_INFO
1368	\| HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
1369	\| HMVMX_READ_GUEST_LINEAR_ADDR
1370	\| HMVMX_READ_GUEST_PHYSICAL_ADDR;
1371	}
1372	#endif
1373
1374	/**
1375	* Verifies that our cached values of the VMCS fields are all consistent with
1376	* what's actually present in the VMCS.
1377	*
1378	* @returns VBox status code.
1379	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
1380	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
1381	* VMCS content. HMCPU error-field is
1382	* updated, see VMX_VCI_XXX.
1383	* @param pVCpu The cross context virtual CPU structure.
1384	* @param pVmcsInfo The VMCS info. object.
1385	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1386	*/
1387	static int vmxHCCheckCachedVmcsCtls(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1388	{
1389	const char * const pcszVmcs = fIsNstGstVmcs ? "Nested-guest VMCS" : "VMCS";
1390
1391	uint32_t u32Val;
1392	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY, &u32Val);
1393	AssertRC(rc);
1394	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
1395	("%s entry controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32EntryCtls, u32Val),
1396	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_ENTRY,
1397	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1398
1399	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_EXIT, &u32Val);
1400	AssertRC(rc);
1401	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
1402	("%s exit controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ExitCtls, u32Val),
1403	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_EXIT,
1404	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1405
1406	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1407	AssertRC(rc);
1408	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
1409	("%s pin controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32PinCtls, u32Val),
1410	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PIN_EXEC,
1411	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1412
1413	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1414	AssertRC(rc);
1415	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
1416	("%s proc controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls, u32Val),
1417	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PROC_EXEC,
1418	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1419
1420	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
1421	{
1422	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1423	AssertRC(rc);
1424	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
1425	("%s proc2 controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls2, u32Val),
1426	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
1427	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1428	}
1429
1430	uint64_t u64Val;
1431	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TERTIARY_CTLS)
1432	{
1433	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_CTRL_PROC_EXEC3_FULL, &u64Val);
1434	AssertRC(rc);
1435	AssertMsgReturnStmt(pVmcsInfo->u64ProcCtls3 == u64Val,
1436	("%s proc3 controls mismatch: Cache=%#RX32 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64ProcCtls3, u64Val),
1437	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PROC_EXEC3,
1438	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1439	}
1440
1441	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
1442	AssertRC(rc);
1443	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
1444	("%s exception bitmap mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32XcptBitmap, u32Val),
1445	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
1446	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1447
1448	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
1449	AssertRC(rc);
1450	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
1451	("%s TSC offset mismatch: Cache=%#RX64 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64TscOffset, u64Val),
1452	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
1453	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1454
1455	NOREF(pcszVmcs);
1456	return VINF_SUCCESS;
1457	}
1458
1459
1460	/**
1461	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
1462	* VMCS.
1463	*
1464	* This is typically required when the guest changes paging mode.
1465	*
1466	* @returns VBox status code.
1467	* @param pVCpu The cross context virtual CPU structure.
1468	* @param pVmxTransient The VMX-transient structure.
1469	*
1470	* @remarks Requires EFER.
1471	* @remarks No-long-jump zone!!!
1472	*/
1473	static int vmxHCExportGuestEntryExitCtls(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
1474	{
1475	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
1476	{
1477	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1478	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1479
1480	/*
1481	* VM-entry controls.
1482	*/
1483	{
1484	uint32_t fVal = g_HmMsrs.u.vmx.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
1485	uint32_t const fZap = g_HmMsrs.u.vmx.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1486
1487	/*
1488	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
1489	* The first VT-x capable CPUs only supported the 1-setting of this bit.
1490	*
1491	* For nested-guests, this is a mandatory VM-entry control. It's also
1492	* required because we do not want to leak host bits to the nested-guest.
1493	*/
1494	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
1495
1496	/*
1497	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
1498	*
1499	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
1500	* required to get the nested-guest working with hardware-assisted VMX execution.
1501	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested hypervisor
1502	* can skip intercepting changes to the EFER MSR. This is why it needs to be done
1503	* here rather than while merging the guest VMCS controls.
1504	*/
1505	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
1506	{
1507	Assert(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LME);
1508	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
1509	}
1510	else
1511	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
1512
1513	/*
1514	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
1515	*
1516	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
1517	* regardless of whether the nested-guest VMCS specifies it because we are free to
1518	* load whatever MSRs we require and we do not need to modify the guest visible copy
1519	* of the VM-entry MSR load area.
1520	*/
1521	if ( g_fHmVmxSupportsVmcsEfer
1522	#ifndef IN_NEM_DARWIN
1523	&& hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient)
1524	#endif
1525	)
1526	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
1527	else
1528	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
1529
1530	/*
1531	* The following should -not- be set (since we're not in SMM mode):
1532	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
1533	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
1534	*/
1535
1536	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
1537	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
1538
1539	if ((fVal & fZap) == fVal)
1540	{ /* likely */ }
1541	else
1542	{
1543	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
1544	g_HmMsrs.u.vmx.EntryCtls.n.allowed0, fVal, fZap));
1545	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_UFC_CTRL_ENTRY;
1546	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1547	}
1548
1549	/* Commit it to the VMCS. */
1550	if (pVmcsInfo->u32EntryCtls != fVal)
1551	{
1552	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY, fVal);
1553	AssertRC(rc);
1554	pVmcsInfo->u32EntryCtls = fVal;
1555	}
1556	}
1557
1558	/*
1559	* VM-exit controls.
1560	*/
1561	{
1562	uint32_t fVal = g_HmMsrs.u.vmx.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
1563	uint32_t const fZap = g_HmMsrs.u.vmx.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1564
1565	/*
1566	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
1567	* supported the 1-setting of this bit.
1568	*
1569	* For nested-guests, we set the "save debug controls" as the converse
1570	* "load debug controls" is mandatory for nested-guests anyway.
1571	*/
1572	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
1573
1574	/*
1575	* Set the host long mode active (EFER.LMA) bit (which Intel calls
1576	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
1577	* host EFER.LMA and EFER.LME bit to this value. See assertion in
1578	* vmxHCExportHostMsrs().
1579	*
1580	* For nested-guests, we always set this bit as we do not support 32-bit
1581	* hosts.
1582	*/
1583	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
1584
1585	#ifndef IN_NEM_DARWIN
1586	/*
1587	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
1588	*
1589	* For nested-guests, we should use the "save IA32_EFER" control if we also
1590	* used the "load IA32_EFER" control while exporting VM-entry controls.
1591	*/
1592	if ( g_fHmVmxSupportsVmcsEfer
1593	&& hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient))
1594	{
1595	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
1596	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
1597	}
1598	#endif
1599
1600	/*
1601	* Enable saving of the VMX-preemption timer value on VM-exit.
1602	* For nested-guests, currently not exposed/used.
1603	*/
1604	/** @todo r=bird: Measure performance hit because of this vs. always rewriting
1605	* the timer value. */
1606	if (VM_IS_VMX_PREEMPT_TIMER_USED(pVM))
1607	{
1608	Assert(g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER);
1609	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
1610	}
1611
1612	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
1613	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
1614
1615	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
1616	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
1617	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
1618
1619	if ((fVal & fZap) == fVal)
1620	{ /* likely */ }
1621	else
1622	{
1623	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
1624	g_HmMsrs.u.vmx.ExitCtls.n.allowed0, fVal, fZap));
1625	VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_UFC_CTRL_EXIT;
1626	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1627	}
1628
1629	/* Commit it to the VMCS. */
1630	if (pVmcsInfo->u32ExitCtls != fVal)
1631	{
1632	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXIT, fVal);
1633	AssertRC(rc);
1634	pVmcsInfo->u32ExitCtls = fVal;
1635	}
1636	}
1637
1638	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
1639	}
1640	return VINF_SUCCESS;
1641	}
1642
1643
1644	/**
1645	* Sets the TPR threshold in the VMCS.
1646	*
1647	* @param pVCpu The cross context virtual CPU structure.
1648	* @param pVmcsInfo The VMCS info. object.
1649	* @param u32TprThreshold The TPR threshold (task-priority class only).
1650	*/
1651	DECLINLINE(void) vmxHCApicSetTprThreshold(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
1652	{
1653	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
1654	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
1655	RT_NOREF(pVmcsInfo);
1656	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
1657	AssertRC(rc);
1658	}
1659
1660
1661	/**
1662	* Exports the guest APIC TPR state into the VMCS.
1663	*
1664	* @param pVCpu The cross context virtual CPU structure.
1665	* @param pVmxTransient The VMX-transient structure.
1666	*
1667	* @remarks No-long-jump zone!!!
1668	*/
1669	static void vmxHCExportGuestApicTpr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
1670	{
1671	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
1672	{
1673	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
1674
1675	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1676	if (!pVmxTransient->fIsNestedGuest)
1677	{
1678	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1679	&& APICIsEnabled(pVCpu))
1680	{
1681	/*
1682	* Setup TPR shadowing.
1683	*/
1684	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
1685	{
1686	bool fPendingIntr = false;
1687	uint8_t u8Tpr = 0;
1688	uint8_t u8PendingIntr = 0;
1689	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
1690	AssertRC(rc);
1691
1692	/*
1693	* If there are interrupts pending but masked by the TPR, instruct VT-x to
1694	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
1695	* priority of the pending interrupt so we can deliver the interrupt. If there
1696	* are no interrupts pending, set threshold to 0 to not cause any
1697	* TPR-below-threshold VM-exits.
1698	*/
1699	uint32_t u32TprThreshold = 0;
1700	if (fPendingIntr)
1701	{
1702	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
1703	(which is the Task-Priority Class). */
1704	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
1705	const uint8_t u8TprPriority = u8Tpr >> 4;
1706	if (u8PendingPriority <= u8TprPriority)
1707	u32TprThreshold = u8PendingPriority;
1708	}
1709
1710	vmxHCApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
1711	}
1712	}
1713	}
1714	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
1715	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
1716	}
1717	}
1718
1719
1720	/**
1721	* Gets the guest interruptibility-state and updates related force-flags.
1722	*
1723	* @returns Guest's interruptibility-state.
1724	* @param pVCpu The cross context virtual CPU structure.
1725	*
1726	* @remarks No-long-jump zone!!!
1727	*/
1728	static uint32_t vmxHCGetGuestIntrStateAndUpdateFFs(PVMCPUCC pVCpu)
1729	{
1730	/*
1731	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
1732	*/
1733	uint32_t fIntrState = 0;
1734	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1735	{
1736	/* If inhibition is active, RIP and RFLAGS should've been imported from the VMCS already. */
1737	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
1738
1739	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1740	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
1741	{
1742	if (pCtx->eflags.Bits.u1IF)
1743	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
1744	else
1745	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
1746	}
1747	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1748	{
1749	/*
1750	* We can clear the inhibit force flag as even if we go back to the recompiler
1751	* without executing guest code in VT-x, the flag's condition to be cleared is
1752	* met and thus the cleared state is correct.
1753	*/
1754	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1755	}
1756	}
1757
1758	/*
1759	* Check if we should inhibit NMI delivery.
1760	*/
1761	if (CPUMIsGuestNmiBlocking(pVCpu))
1762	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
1763
1764	/*
1765	* Validate.
1766	*/
1767	#ifdef VBOX_STRICT
1768	/* We don't support block-by-SMI yet.*/
1769	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI));
1770
1771	/* Block-by-STI must not be set when interrupts are disabled. */
1772	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
1773	{
1774	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
1775	Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_IF);
1776	}
1777	#endif
1778
1779	return fIntrState;
1780	}
1781
1782
1783	/**
1784	* Exports the exception intercepts required for guest execution in the VMCS.
1785	*
1786	* @param pVCpu The cross context virtual CPU structure.
1787	* @param pVmxTransient The VMX-transient structure.
1788	*
1789	* @remarks No-long-jump zone!!!
1790	*/
1791	static void vmxHCExportGuestXcptIntercepts(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
1792	{
1793	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS)
1794	{
1795	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
1796	if ( !pVmxTransient->fIsNestedGuest
1797	&& VCPU_2_VMXSTATE(pVCpu).fGIMTrapXcptUD)
1798	vmxHCAddXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
1799	else
1800	vmxHCRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
1801
1802	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
1803	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS);
1804	}
1805	}
1806
1807
1808	/**
1809	* Exports the guest's RIP into the guest-state area in the VMCS.
1810	*
1811	* @param pVCpu The cross context virtual CPU structure.
1812	*
1813	* @remarks No-long-jump zone!!!
1814	*/
1815	static void vmxHCExportGuestRip(PVMCPUCC pVCpu)
1816	{
1817	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_RIP)
1818	{
1819	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
1820
1821	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
1822	AssertRC(rc);
1823
1824	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_RIP);
1825	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
1826	}
1827	}
1828
1829
1830	/**
1831	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
1832	*
1833	* @param pVCpu The cross context virtual CPU structure.
1834	* @param pVmxTransient The VMX-transient structure.
1835	*
1836	* @remarks No-long-jump zone!!!
1837	*/
1838	static void vmxHCExportGuestRflags(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
1839	{
1840	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
1841	{
1842	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
1843
1844	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
1845	Let us assert it as such and use 32-bit VMWRITE. */
1846	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
1847	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
1848	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
1849	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
1850
1851	#ifndef IN_NEM_DARWIN
1852	/*
1853	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
1854	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
1855	* can run the real-mode guest code under Virtual 8086 mode.
1856	*/
1857	PVMXVMCSINFOSHARED pVmcsInfo = pVmxTransient->pVmcsInfo->pShared;
1858	if (pVmcsInfo->RealMode.fRealOnV86Active)
1859	{
1860	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
1861	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
1862	Assert(!pVmxTransient->fIsNestedGuest);
1863	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
1864	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
1865	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
1866	}
1867	#else
1868	RT_NOREF(pVmxTransient);
1869	#endif
1870
1871	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
1872	AssertRC(rc);
1873
1874	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
1875	Log4Func(("eflags=%#RX32\n", fEFlags.u32));
1876	}
1877	}
1878
1879
1880	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1881	/**
1882	* Copies the nested-guest VMCS to the shadow VMCS.
1883	*
1884	* @returns VBox status code.
1885	* @param pVCpu The cross context virtual CPU structure.
1886	* @param pVmcsInfo The VMCS info. object.
1887	*
1888	* @remarks No-long-jump zone!!!
1889	*/
1890	static int vmxHCCopyNstGstToShadowVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
1891	{
1892	PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
1893	PCVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
1894
1895	/*
1896	* Disable interrupts so we don't get preempted while the shadow VMCS is the
1897	* current VMCS, as we may try saving guest lazy MSRs.
1898	*
1899	* Strictly speaking the lazy MSRs are not in the VMCS, but I'd rather not risk
1900	* calling the import VMCS code which is currently performing the guest MSR reads
1901	* (on 64-bit hosts) and accessing the auto-load/store MSR area on 32-bit hosts
1902	* and the rest of the VMX leave session machinery.
1903	*/
1904	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
1905
1906	int rc = vmxHCLoadShadowVmcs(pVmcsInfo);
1907	if (RT_SUCCESS(rc))
1908	{
1909	/*
1910	* Copy all guest read/write VMCS fields.
1911	*
1912	* We don't check for VMWRITE failures here for performance reasons and
1913	* because they are not expected to fail, barring irrecoverable conditions
1914	* like hardware errors.
1915	*/
1916	uint32_t const cShadowVmcsFields = pVM->hmr0.s.vmx.cShadowVmcsFields;
1917	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
1918	{
1919	uint64_t u64Val;
1920	uint32_t const uVmcsField = pVM->hmr0.s.vmx.paShadowVmcsFields[i];
1921	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
1922	VMX_VMCS_WRITE_64(pVCpu, uVmcsField, u64Val);
1923	}
1924
1925	/*
1926	* If the host CPU supports writing all VMCS fields, copy the guest read-only
1927	* VMCS fields, so the guest can VMREAD them without causing a VM-exit.
1928	*/
1929	if (g_HmMsrs.u.vmx.u64Misc & VMX_MISC_VMWRITE_ALL)
1930	{
1931	uint32_t const cShadowVmcsRoFields = pVM->hmr0.s.vmx.cShadowVmcsRoFields;
1932	for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
1933	{
1934	uint64_t u64Val;
1935	uint32_t const uVmcsField = pVM->hmr0.s.vmx.paShadowVmcsRoFields[i];
1936	IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
1937	VMX_VMCS_WRITE_64(pVCpu, uVmcsField, u64Val);
1938	}
1939	}
1940
1941	rc = vmxHCClearShadowVmcs(pVmcsInfo);
1942	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
1943	}
1944
1945	ASMSetFlags(fEFlags);
1946	return rc;
1947	}
1948
1949
1950	/**
1951	* Copies the shadow VMCS to the nested-guest VMCS.
1952	*
1953	* @returns VBox status code.
1954	* @param pVCpu The cross context virtual CPU structure.
1955	* @param pVmcsInfo The VMCS info. object.
1956	*
1957	* @remarks Called with interrupts disabled.
1958	*/
1959	static int vmxHCCopyShadowToNstGstVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
1960	{
1961	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1962	PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
1963	PVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
1964
1965	int rc = vmxHCLoadShadowVmcs(pVmcsInfo);
1966	if (RT_SUCCESS(rc))
1967	{
1968	/*
1969	* Copy guest read/write fields from the shadow VMCS.
1970	* Guest read-only fields cannot be modified, so no need to copy them.
1971	*
1972	* We don't check for VMREAD failures here for performance reasons and
1973	* because they are not expected to fail, barring irrecoverable conditions
1974	* like hardware errors.
1975	*/
1976	uint32_t const cShadowVmcsFields = pVM->hmr0.s.vmx.cShadowVmcsFields;
1977	for (uint32_t i = 0; i < cShadowVmcsFields; i++)
1978	{
1979	uint64_t u64Val;
1980	uint32_t const uVmcsField = pVM->hmr0.s.vmx.paShadowVmcsFields[i];
1981	VMX_VMCS_READ_64(pVCpu, uVmcsField, &u64Val);
1982	IEMWriteVmxVmcsField(pVmcsNstGst, uVmcsField, u64Val);
1983	}
1984
1985	rc = vmxHCClearShadowVmcs(pVmcsInfo);
1986	rc \|= hmR0VmxLoadVmcs(pVmcsInfo);
1987	}
1988	return rc;
1989	}
1990
1991
1992	/**
1993	* Enables VMCS shadowing for the given VMCS info. object.
1994	*
1995	* @param pVCpu The cross context virtual CPU structure.
1996	* @param pVmcsInfo The VMCS info. object.
1997	*
1998	* @remarks No-long-jump zone!!!
1999	*/
2000	static void vmxHCEnableVmcsShadowing(PCVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2001	{
2002	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
2003	if (!(uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING))
2004	{
2005	Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
2006	uProcCtls2 \|= VMX_PROC_CTLS2_VMCS_SHADOWING;
2007	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2); AssertRC(rc);
2008	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, pVmcsInfo->HCPhysShadowVmcs); AssertRC(rc);
2009	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
2010	pVmcsInfo->u64VmcsLinkPtr = pVmcsInfo->HCPhysShadowVmcs;
2011	Log4Func(("Enabled\n"));
2012	}
2013	}
2014
2015
2016	/**
2017	* Disables VMCS shadowing for the given VMCS info. object.
2018	*
2019	* @param pVCpu The cross context virtual CPU structure.
2020	* @param pVmcsInfo The VMCS info. object.
2021	*
2022	* @remarks No-long-jump zone!!!
2023	*/
2024	static void vmxHCDisableVmcsShadowing(PCVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2025	{
2026	/*
2027	* We want all VMREAD and VMWRITE instructions to cause VM-exits, so we clear the
2028	* VMCS shadowing control. However, VM-entry requires the shadow VMCS indicator bit
2029	* to match the VMCS shadowing control if the VMCS link pointer is not NIL_RTHCPHYS.
2030	* Hence, we must also reset the VMCS link pointer to ensure VM-entry does not fail.
2031	*
2032	* See Intel spec. 26.2.1.1 "VM-Execution Control Fields".
2033	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
2034	*/
2035	uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
2036	if (uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
2037	{
2038	uProcCtls2 &= ~VMX_PROC_CTLS2_VMCS_SHADOWING;
2039	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2); AssertRC(rc);
2040	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS); AssertRC(rc);
2041	pVmcsInfo->u32ProcCtls2 = uProcCtls2;
2042	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
2043	Log4Func(("Disabled\n"));
2044	}
2045	}
2046	#endif
2047
2048
2049	/**
2050	* Exports the guest CR0 control register into the guest-state area in the VMCS.
2051	*
2052	* The guest FPU state is always pre-loaded hence we don't need to bother about
2053	* sharing FPU related CR0 bits between the guest and host.
2054	*
2055	* @returns VBox status code.
2056	* @param pVCpu The cross context virtual CPU structure.
2057	* @param pVmxTransient The VMX-transient structure.
2058	*
2059	* @remarks No-long-jump zone!!!
2060	*/
2061	static int vmxHCExportGuestCR0(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
2062	{
2063	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CR0)
2064	{
2065	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2066	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2067
2068	uint64_t fSetCr0 = g_HmMsrs.u.vmx.u64Cr0Fixed0;
2069	uint64_t const fZapCr0 = g_HmMsrs.u.vmx.u64Cr0Fixed1;
2070	if (VM_IS_VMX_UNRESTRICTED_GUEST(pVM))
2071	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
2072	else
2073	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2074
2075	if (!pVmxTransient->fIsNestedGuest)
2076	{
2077	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
2078	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
2079	uint64_t const u64ShadowCr0 = u64GuestCr0;
2080	Assert(!RT_HI_U32(u64GuestCr0));
2081
2082	/*
2083	* Setup VT-x's view of the guest CR0.
2084	*/
2085	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
2086	if (VM_IS_VMX_NESTED_PAGING(pVM))
2087	{
2088	#ifndef HMVMX_ALAWAYS_INTERCEPT_CR3_ACCESS
2089	if (CPUMIsGuestPagingEnabled(pVCpu))
2090	{
2091	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
2092	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
2093	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
2094	}
2095	else
2096	{
2097	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
2098	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
2099	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
2100	}
2101
2102	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
2103	if (VM_IS_VMX_UNRESTRICTED_GUEST(pVM))
2104	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
2105	#endif
2106	}
2107	else
2108	{
2109	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
2110	u64GuestCr0 \|= X86_CR0_WP;
2111	}
2112
2113	/*
2114	* Guest FPU bits.
2115	*
2116	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
2117	* using CR0.TS.
2118	*
2119	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
2120	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2121	*/
2122	u64GuestCr0 \|= X86_CR0_NE;
2123
2124	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
2125	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
2126
2127	/*
2128	* Update exception intercepts.
2129	*/
2130	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
2131	#ifndef IN_NEM_DARWIN
2132	if (pVmcsInfo->pShared->RealMode.fRealOnV86Active)
2133	{
2134	Assert(PDMVmmDevHeapIsEnabled(pVM));
2135	Assert(pVM->hm.s.vmx.pRealModeTSS);
2136	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
2137	}
2138	else
2139	#endif
2140	{
2141	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
2142	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
2143	if (fInterceptMF)
2144	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
2145	}
2146
2147	/* Additional intercepts for debugging, define these yourself explicitly. */
2148	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
2149	uXcptBitmap \|= 0
2150	\| RT_BIT(X86_XCPT_BP)
2151	\| RT_BIT(X86_XCPT_DE)
2152	\| RT_BIT(X86_XCPT_NM)
2153	\| RT_BIT(X86_XCPT_TS)
2154	\| RT_BIT(X86_XCPT_UD)
2155	\| RT_BIT(X86_XCPT_NP)
2156	\| RT_BIT(X86_XCPT_SS)
2157	\| RT_BIT(X86_XCPT_GP)
2158	\| RT_BIT(X86_XCPT_PF)
2159	\| RT_BIT(X86_XCPT_MF)
2160	;
2161	#elif defined(HMVMX_ALWAYS_TRAP_PF)
2162	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
2163	#endif
2164	if (VCPU_2_VMXSTATE(pVCpu).fTrapXcptGpForLovelyMesaDrv)
2165	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
2166	Assert(VM_IS_VMX_NESTED_PAGING(pVM) \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
2167
2168	/* Apply the hardware specified CR0 fixed bits and enable caching. */
2169	u64GuestCr0 \|= fSetCr0;
2170	u64GuestCr0 &= fZapCr0;
2171	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
2172
2173	/* Commit the CR0 and related fields to the guest VMCS. */
2174	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR0, u64GuestCr0); AssertRC(rc);
2175	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0); AssertRC(rc);
2176	if (uProcCtls != pVmcsInfo->u32ProcCtls)
2177	{
2178	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
2179	AssertRC(rc);
2180	}
2181	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
2182	{
2183	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
2184	AssertRC(rc);
2185	}
2186
2187	/* Update our caches. */
2188	pVmcsInfo->u32ProcCtls = uProcCtls;
2189	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
2190
2191	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
2192	}
2193	else
2194	{
2195	/*
2196	* With nested-guests, we may have extended the guest/host mask here since we
2197	* merged in the outer guest's mask. Thus, the merged mask can include more bits
2198	* (to read from the nested-guest CR0 read-shadow) than the nested hypervisor
2199	* originally supplied. We must copy those bits from the nested-guest CR0 into
2200	* the nested-guest CR0 read-shadow.
2201	*/
2202	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
2203	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
2204	uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(&pVCpu->cpum.GstCtx, pVmcsInfo->u64Cr0Mask);
2205	Assert(!RT_HI_U32(u64GuestCr0));
2206	Assert(u64GuestCr0 & X86_CR0_NE);
2207
2208	/* Apply the hardware specified CR0 fixed bits and enable caching. */
2209	u64GuestCr0 \|= fSetCr0;
2210	u64GuestCr0 &= fZapCr0;
2211	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
2212
2213	/* Commit the CR0 and CR0 read-shadow to the nested-guest VMCS. */
2214	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR0, u64GuestCr0); AssertRC(rc);
2215	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0); AssertRC(rc);
2216
2217	Log4Func(("cr0=%#RX64 shadow=%#RX64 (set=%#RX64 zap=%#RX64)\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
2218	}
2219
2220	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CR0);
2221	}
2222
2223	return VINF_SUCCESS;
2224	}
2225
2226
2227	/**
2228	* Exports the guest control registers (CR3, CR4) into the guest-state area
2229	* in the VMCS.
2230	*
2231	* @returns VBox strict status code.
2232	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
2233	* without unrestricted guest access and the VMMDev is not presently
2234	* mapped (e.g. EFI32).
2235	*
2236	* @param pVCpu The cross context virtual CPU structure.
2237	* @param pVmxTransient The VMX-transient structure.
2238	*
2239	* @remarks No-long-jump zone!!!
2240	*/
2241	static VBOXSTRICTRC vmxHCExportGuestCR3AndCR4(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
2242	{
2243	int rc = VINF_SUCCESS;
2244	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2245
2246	/*
2247	* Guest CR2.
2248	* It's always loaded in the assembler code. Nothing to do here.
2249	*/
2250
2251	/*
2252	* Guest CR3.
2253	*/
2254	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CR3)
2255	{
2256	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
2257
2258	if (VM_IS_VMX_NESTED_PAGING(pVM))
2259	{
2260	#ifndef IN_NEM_DARWIN
2261	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2262	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
2263
2264	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
2265	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
2266	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
2267	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
2268
2269	/* VMX_EPT_MEMTYPE_WB support is already checked in vmxHCSetupTaggedTlb(). */
2270	pVmcsInfo->HCPhysEPTP \|= RT_BF_MAKE(VMX_BF_EPTP_MEMTYPE, VMX_EPTP_MEMTYPE_WB)
2271	\| RT_BF_MAKE(VMX_BF_EPTP_PAGE_WALK_LENGTH, VMX_EPTP_PAGE_WALK_LENGTH_4);
2272
2273	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
2274	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
2275	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
2276	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
2277	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
2278	\|\| (g_HmMsrs.u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_ACCESS_DIRTY),
2279	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
2280
2281	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
2282	AssertRC(rc);
2283	#endif
2284
2285	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2286	uint64_t u64GuestCr3 = pCtx->cr3;
2287	if ( VM_IS_VMX_UNRESTRICTED_GUEST(pVM)
2288	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
2289	{
2290	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
2291	if (CPUMIsGuestInPAEModeEx(pCtx))
2292	{
2293	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE0_FULL, pCtx->aPaePdpes[0].u); AssertRC(rc);
2294	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE1_FULL, pCtx->aPaePdpes[1].u); AssertRC(rc);
2295	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE2_FULL, pCtx->aPaePdpes[2].u); AssertRC(rc);
2296	rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE3_FULL, pCtx->aPaePdpes[3].u); AssertRC(rc);
2297	}
2298
2299	/*
2300	* The guest's view of its CR3 is unblemished with nested paging when the
2301	* guest is using paging or we have unrestricted guest execution to handle
2302	* the guest when it's not using paging.
2303	*/
2304	}
2305	#ifndef IN_NEM_DARWIN
2306	else
2307	{
2308	/*
2309	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
2310	* thinks it accesses physical memory directly, we use our identity-mapped
2311	* page table to map guest-linear to guest-physical addresses. EPT takes care
2312	* of translating it to host-physical addresses.
2313	*/
2314	RTGCPHYS GCPhys;
2315	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
2316
2317	/* We obtain it here every time as the guest could have relocated this PCI region. */
2318	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2319	if (RT_SUCCESS(rc))
2320	{ /* likely */ }
2321	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
2322	{
2323	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
2324	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
2325	}
2326	else
2327	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
2328
2329	u64GuestCr3 = GCPhys;
2330	}
2331	#endif
2332
2333	Log4Func(("guest_cr3=%#RX64 (GstN)\n", u64GuestCr3));
2334	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR3, u64GuestCr3);
2335	AssertRC(rc);
2336	}
2337	else
2338	{
2339	Assert(!pVmxTransient->fIsNestedGuest);
2340	/* Non-nested paging case, just use the hypervisor's CR3. */
2341	RTHCPHYS const HCPhysGuestCr3 = PGMGetHyperCR3(pVCpu);
2342
2343	Log4Func(("guest_cr3=%#RX64 (HstN)\n", HCPhysGuestCr3));
2344	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR3, HCPhysGuestCr3);
2345	AssertRC(rc);
2346	}
2347
2348	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CR3);
2349	}
2350
2351	/*
2352	* Guest CR4.
2353	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
2354	*/
2355	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CR4)
2356	{
2357	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2358	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2359
2360	uint64_t const fSetCr4 = g_HmMsrs.u.vmx.u64Cr4Fixed0;
2361	uint64_t const fZapCr4 = g_HmMsrs.u.vmx.u64Cr4Fixed1;
2362
2363	/*
2364	* With nested-guests, we may have extended the guest/host mask here (since we
2365	* merged in the outer guest's mask, see vmxHCMergeVmcsNested). This means, the
2366	* mask can include more bits (to read from the nested-guest CR4 read-shadow) than
2367	* the nested hypervisor originally supplied. Thus, we should, in essence, copy
2368	* those bits from the nested-guest CR4 into the nested-guest CR4 read-shadow.
2369	*/
2370	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
2371	uint64_t u64GuestCr4 = pCtx->cr4;
2372	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest
2373	? pCtx->cr4
2374	: CPUMGetGuestVmxMaskedCr4(pCtx, pVmcsInfo->u64Cr4Mask);
2375	Assert(!RT_HI_U32(u64GuestCr4));
2376
2377	#ifndef IN_NEM_DARWIN
2378	/*
2379	* Setup VT-x's view of the guest CR4.
2380	*
2381	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
2382	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
2383	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
2384	*
2385	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
2386	*/
2387	if (pVmcsInfo->pShared->RealMode.fRealOnV86Active)
2388	{
2389	Assert(pVM->hm.s.vmx.pRealModeTSS);
2390	Assert(PDMVmmDevHeapIsEnabled(pVM));
2391	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
2392	}
2393	#endif
2394
2395	if (VM_IS_VMX_NESTED_PAGING(pVM))
2396	{
2397	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
2398	&& !VM_IS_VMX_UNRESTRICTED_GUEST(pVM))
2399	{
2400	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
2401	u64GuestCr4 \|= X86_CR4_PSE;
2402	/* Our identity mapping is a 32-bit page directory. */
2403	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
2404	}
2405	/* else use guest CR4.*/
2406	}
2407	else
2408	{
2409	Assert(!pVmxTransient->fIsNestedGuest);
2410
2411	/*
2412	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
2413	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
2414	*/
2415	switch (VCPU_2_VMXSTATE(pVCpu).enmShadowMode)
2416	{
2417	case PGMMODE_REAL: /* Real-mode. */
2418	case PGMMODE_PROTECTED: /* Protected mode without paging. */
2419	case PGMMODE_32_BIT: /* 32-bit paging. */
2420	{
2421	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
2422	break;
2423	}
2424
2425	case PGMMODE_PAE: /* PAE paging. */
2426	case PGMMODE_PAE_NX: /* PAE paging with NX. */
2427	{
2428	u64GuestCr4 \|= X86_CR4_PAE;
2429	break;
2430	}
2431
2432	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
2433	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
2434	{
2435	#ifdef VBOX_WITH_64_BITS_GUESTS
2436	/* For our assumption in vmxHCShouldSwapEferMsr. */
2437	Assert(u64GuestCr4 & X86_CR4_PAE);
2438	break;
2439	#endif
2440	}
2441	default:
2442	AssertFailed();
2443	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2444	}
2445	}
2446
2447	/* Apply the hardware specified CR4 fixed bits (mainly CR4.VMXE). */
2448	u64GuestCr4 \|= fSetCr4;
2449	u64GuestCr4 &= fZapCr4;
2450
2451	/* Commit the CR4 and CR4 read-shadow to the guest VMCS. */
2452	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR4, u64GuestCr4); AssertRC(rc);
2453	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4); AssertRC(rc);
2454
2455	#ifndef IN_NEM_DARWIN
2456	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
2457	bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
2458	if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
2459	{
2460	pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
2461	hmR0VmxUpdateStartVmFunction(pVCpu);
2462	}
2463	#endif
2464
2465	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CR4);
2466
2467	Log4Func(("cr4=%#RX64 shadow=%#RX64 (set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
2468	}
2469	return rc;
2470	}
2471
2472
2473	#ifdef VBOX_STRICT
2474	/**
2475	* Strict function to validate segment registers.
2476	*
2477	* @param pVCpu The cross context virtual CPU structure.
2478	* @param pVmcsInfo The VMCS info. object.
2479	*
2480	* @remarks Will import guest CR0 on strict builds during validation of
2481	* segments.
2482	*/
2483	static void vmxHCValidateSegmentRegs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2484	{
2485	/*
2486	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
2487	*
2488	* The reason we check for attribute value 0 in this function and not just the unusable bit is
2489	* because vmxHCExportGuestSegReg() only updates the VMCS' copy of the value with the
2490	* unusable bit and doesn't change the guest-context value.
2491	*/
2492	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2493	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2494	vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
2495	if ( !VM_IS_VMX_UNRESTRICTED_GUEST(pVM)
2496	&& ( !CPUMIsGuestInRealModeEx(pCtx)
2497	&& !CPUMIsGuestInV86ModeEx(pCtx)))
2498	{
2499	/* Protected mode checks */
2500	/* CS */
2501	Assert(pCtx->cs.Attr.n.u1Present);
2502	Assert(!(pCtx->cs.Attr.u & 0xf00));
2503	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
2504	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
2505	\|\| !(pCtx->cs.Attr.n.u1Granularity));
2506	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
2507	\|\| (pCtx->cs.Attr.n.u1Granularity));
2508	/* CS cannot be loaded with NULL in protected mode. */
2509	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
2510	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
2511	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
2512	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
2513	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
2514	else
2515	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
2516	/* SS */
2517	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
2518	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
2519	if ( !(pCtx->cr0 & X86_CR0_PE)
2520	\|\| pCtx->cs.Attr.n.u4Type == 3)
2521	{
2522	Assert(!pCtx->ss.Attr.n.u2Dpl);
2523	}
2524	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
2525	{
2526	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
2527	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
2528	Assert(pCtx->ss.Attr.n.u1Present);
2529	Assert(!(pCtx->ss.Attr.u & 0xf00));
2530	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
2531	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
2532	\|\| !(pCtx->ss.Attr.n.u1Granularity));
2533	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
2534	\|\| (pCtx->ss.Attr.n.u1Granularity));
2535	}
2536	/* DS, ES, FS, GS - only check for usable selectors, see vmxHCExportGuestSegReg(). */
2537	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
2538	{
2539	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
2540	Assert(pCtx->ds.Attr.n.u1Present);
2541	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
2542	Assert(!(pCtx->ds.Attr.u & 0xf00));
2543	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
2544	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
2545	\|\| !(pCtx->ds.Attr.n.u1Granularity));
2546	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
2547	\|\| (pCtx->ds.Attr.n.u1Granularity));
2548	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
2549	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
2550	}
2551	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
2552	{
2553	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
2554	Assert(pCtx->es.Attr.n.u1Present);
2555	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
2556	Assert(!(pCtx->es.Attr.u & 0xf00));
2557	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
2558	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
2559	\|\| !(pCtx->es.Attr.n.u1Granularity));
2560	Assert( !(pCtx->es.u32Limit & 0xfff00000)
2561	\|\| (pCtx->es.Attr.n.u1Granularity));
2562	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
2563	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
2564	}
2565	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
2566	{
2567	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
2568	Assert(pCtx->fs.Attr.n.u1Present);
2569	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
2570	Assert(!(pCtx->fs.Attr.u & 0xf00));
2571	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
2572	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
2573	\|\| !(pCtx->fs.Attr.n.u1Granularity));
2574	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
2575	\|\| (pCtx->fs.Attr.n.u1Granularity));
2576	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
2577	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
2578	}
2579	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
2580	{
2581	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
2582	Assert(pCtx->gs.Attr.n.u1Present);
2583	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
2584	Assert(!(pCtx->gs.Attr.u & 0xf00));
2585	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
2586	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
2587	\|\| !(pCtx->gs.Attr.n.u1Granularity));
2588	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
2589	\|\| (pCtx->gs.Attr.n.u1Granularity));
2590	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
2591	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
2592	}
2593	/* 64-bit capable CPUs. */
2594	Assert(!RT_HI_U32(pCtx->cs.u64Base));
2595	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
2596	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
2597	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
2598	}
2599	else if ( CPUMIsGuestInV86ModeEx(pCtx)
2600	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
2601	&& !VM_IS_VMX_UNRESTRICTED_GUEST(pVM)))
2602	{
2603	/* Real and v86 mode checks. */
2604	/* vmxHCExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
2605	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
2606	#ifndef IN_NEM_DARWIN
2607	if (pVmcsInfo->pShared->RealMode.fRealOnV86Active)
2608	{
2609	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
2610	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
2611	}
2612	else
2613	#endif
2614	{
2615	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
2616	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
2617	}
2618
2619	/* CS */
2620	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
2621	Assert(pCtx->cs.u32Limit == 0xffff);
2622	Assert(u32CSAttr == 0xf3);
2623	/* SS */
2624	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
2625	Assert(pCtx->ss.u32Limit == 0xffff);
2626	Assert(u32SSAttr == 0xf3);
2627	/* DS */
2628	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
2629	Assert(pCtx->ds.u32Limit == 0xffff);
2630	Assert(u32DSAttr == 0xf3);
2631	/* ES */
2632	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
2633	Assert(pCtx->es.u32Limit == 0xffff);
2634	Assert(u32ESAttr == 0xf3);
2635	/* FS */
2636	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
2637	Assert(pCtx->fs.u32Limit == 0xffff);
2638	Assert(u32FSAttr == 0xf3);
2639	/* GS */
2640	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
2641	Assert(pCtx->gs.u32Limit == 0xffff);
2642	Assert(u32GSAttr == 0xf3);
2643	/* 64-bit capable CPUs. */
2644	Assert(!RT_HI_U32(pCtx->cs.u64Base));
2645	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
2646	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
2647	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
2648	}
2649	}
2650	#endif /* VBOX_STRICT */
2651
2652
2653	/**
2654	* Exports a guest segment register into the guest-state area in the VMCS.
2655	*
2656	* @returns VBox status code.
2657	* @param pVCpu The cross context virtual CPU structure.
2658	* @param pVmcsInfo The VMCS info. object.
2659	* @param iSegReg The segment register number (X86_SREG_XXX).
2660	* @param pSelReg Pointer to the segment selector.
2661	*
2662	* @remarks No-long-jump zone!!!
2663	*/
2664	static int vmxHCExportGuestSegReg(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, uint32_t iSegReg, PCCPUMSELREG pSelReg)
2665	{
2666	Assert(iSegReg < X86_SREG_COUNT);
2667
2668	uint32_t u32Access = pSelReg->Attr.u;
2669	#ifndef IN_NEM_DARWIN
2670	if (!pVmcsInfo->pShared->RealMode.fRealOnV86Active)
2671	#endif
2672	{
2673	/*
2674	* The way to differentiate between whether this is really a null selector or was just
2675	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
2676	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
2677	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
2678	* NULL selectors loaded in protected-mode have their attribute as 0.
2679	*/
2680	if (u32Access)
2681	{ }
2682	else
2683	u32Access = X86DESCATTR_UNUSABLE;
2684	}
2685	#ifndef IN_NEM_DARWIN
2686	else
2687	{
2688	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
2689	u32Access = 0xf3;
2690	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2691	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2692	RT_NOREF_PV(pVCpu);
2693	}
2694	#else
2695	RT_NOREF(pVmcsInfo);
2696	#endif
2697
2698	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
2699	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
2700	("Access bit not set for usable segment. %.2s sel=%#x attr %#x\n", "ESCSSSDSFSGS" + iSegReg * 2, pSelReg, pSelReg->Attr.u));
2701
2702	/*
2703	* Commit it to the VMCS.
2704	*/
2705	Assert((uint32_t)VMX_VMCS16_GUEST_SEG_SEL(iSegReg) == g_aVmcsSegSel[iSegReg]);
2706	Assert((uint32_t)VMX_VMCS32_GUEST_SEG_LIMIT(iSegReg) == g_aVmcsSegLimit[iSegReg]);
2707	Assert((uint32_t)VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(iSegReg) == g_aVmcsSegAttr[iSegReg]);
2708	Assert((uint32_t)VMX_VMCS_GUEST_SEG_BASE(iSegReg) == g_aVmcsSegBase[iSegReg]);
2709	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS16_GUEST_SEG_SEL(iSegReg), pSelReg->Sel); AssertRC(rc);
2710	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_SEG_LIMIT(iSegReg), pSelReg->u32Limit); AssertRC(rc);
2711	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_SEG_BASE(iSegReg), pSelReg->u64Base); AssertRC(rc);
2712	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(iSegReg), u32Access); AssertRC(rc);
2713	return VINF_SUCCESS;
2714	}
2715
2716
2717	/**
2718	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
2719	* area in the VMCS.
2720	*
2721	* @returns VBox status code.
2722	* @param pVCpu The cross context virtual CPU structure.
2723	* @param pVmxTransient The VMX-transient structure.
2724	*
2725	* @remarks Will import guest CR0 on strict builds during validation of
2726	* segments.
2727	* @remarks No-long-jump zone!!!
2728	*/
2729	static int vmxHCExportGuestSegRegsXdtr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
2730	{
2731	int rc = VERR_INTERNAL_ERROR_5;
2732	#ifndef IN_NEM_DARWIN
2733	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2734	#endif
2735	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2736	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
2737	#ifndef IN_NEM_DARWIN
2738	PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
2739	#endif
2740
2741	/*
2742	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
2743	*/
2744	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
2745	{
2746	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CS)
2747	{
2748	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
2749	#ifndef IN_NEM_DARWIN
2750	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
2751	pVmcsInfoShared->RealMode.AttrCS.u = pCtx->cs.Attr.u;
2752	#endif
2753	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
2754	AssertRC(rc);
2755	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CS);
2756	}
2757
2758	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_SS)
2759	{
2760	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
2761	#ifndef IN_NEM_DARWIN
2762	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
2763	pVmcsInfoShared->RealMode.AttrSS.u = pCtx->ss.Attr.u;
2764	#endif
2765	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
2766	AssertRC(rc);
2767	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_SS);
2768	}
2769
2770	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_DS)
2771	{
2772	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
2773	#ifndef IN_NEM_DARWIN
2774	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
2775	pVmcsInfoShared->RealMode.AttrDS.u = pCtx->ds.Attr.u;
2776	#endif
2777	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
2778	AssertRC(rc);
2779	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_DS);
2780	}
2781
2782	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_ES)
2783	{
2784	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
2785	#ifndef IN_NEM_DARWIN
2786	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
2787	pVmcsInfoShared->RealMode.AttrES.u = pCtx->es.Attr.u;
2788	#endif
2789	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
2790	AssertRC(rc);
2791	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_ES);
2792	}
2793
2794	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_FS)
2795	{
2796	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
2797	#ifndef IN_NEM_DARWIN
2798	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
2799	pVmcsInfoShared->RealMode.AttrFS.u = pCtx->fs.Attr.u;
2800	#endif
2801	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
2802	AssertRC(rc);
2803	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_FS);
2804	}
2805
2806	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_GS)
2807	{
2808	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
2809	#ifndef IN_NEM_DARWIN
2810	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
2811	pVmcsInfoShared->RealMode.AttrGS.u = pCtx->gs.Attr.u;
2812	#endif
2813	rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
2814	AssertRC(rc);
2815	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_GS);
2816	}
2817
2818	#ifdef VBOX_STRICT
2819	vmxHCValidateSegmentRegs(pVCpu, pVmcsInfo);
2820	#endif
2821	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
2822	pCtx->cs.Attr.u));
2823	}
2824
2825	/*
2826	* Guest TR.
2827	*/
2828	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_TR)
2829	{
2830	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
2831
2832	/*
2833	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
2834	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
2835	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
2836	*/
2837	uint16_t u16Sel;
2838	uint32_t u32Limit;
2839	uint64_t u64Base;
2840	uint32_t u32AccessRights;
2841	#ifndef IN_NEM_DARWIN
2842	if (!pVmcsInfoShared->RealMode.fRealOnV86Active)
2843	#endif
2844	{
2845	u16Sel = pCtx->tr.Sel;
2846	u32Limit = pCtx->tr.u32Limit;
2847	u64Base = pCtx->tr.u64Base;
2848	u32AccessRights = pCtx->tr.Attr.u;
2849	}
2850	#ifndef IN_NEM_DARWIN
2851	else
2852	{
2853	Assert(!pVmxTransient->fIsNestedGuest);
2854	Assert(pVM->hm.s.vmx.pRealModeTSS);
2855	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
2856
2857	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
2858	RTGCPHYS GCPhys;
2859	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
2860	AssertRCReturn(rc, rc);
2861
2862	X86DESCATTR DescAttr;
2863	DescAttr.u = 0;
2864	DescAttr.n.u1Present = 1;
2865	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
2866
2867	u16Sel = 0;
2868	u32Limit = HM_VTX_TSS_SIZE;
2869	u64Base = GCPhys;
2870	u32AccessRights = DescAttr.u;
2871	}
2872	#endif
2873
2874	/* Validate. */
2875	Assert(!(u16Sel & RT_BIT(2)));
2876	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
2877	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
2878	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
2879	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
2880	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
2881	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
2882	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
2883	Assert( (u32Limit & 0xfff) == 0xfff
2884	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
2885	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
2886	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
2887
2888	rc = VMX_VMCS_WRITE_16(pVCpu, VMX_VMCS16_GUEST_TR_SEL, u16Sel); AssertRC(rc);
2889	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRC(rc);
2890	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRC(rc);
2891	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRC(rc);
2892
2893	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_TR);
2894	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
2895	}
2896
2897	/*
2898	* Guest GDTR.
2899	*/
2900	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_GDTR)
2901	{
2902	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
2903
2904	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt); AssertRC(rc);
2905	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt); AssertRC(rc);
2906
2907	/* Validate. */
2908	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
2909
2910	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
2911	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
2912	}
2913
2914	/*
2915	* Guest LDTR.
2916	*/
2917	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_LDTR)
2918	{
2919	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
2920
2921	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
2922	uint32_t u32Access;
2923	if ( !pVmxTransient->fIsNestedGuest
2924	&& !pCtx->ldtr.Attr.u)
2925	u32Access = X86DESCATTR_UNUSABLE;
2926	else
2927	u32Access = pCtx->ldtr.Attr.u;
2928
2929	rc = VMX_VMCS_WRITE_16(pVCpu, VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel); AssertRC(rc);
2930	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit); AssertRC(rc);
2931	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRC(rc);
2932	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base); AssertRC(rc);
2933
2934	/* Validate. */
2935	if (!(u32Access & X86DESCATTR_UNUSABLE))
2936	{
2937	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
2938	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
2939	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
2940	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
2941	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
2942	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
2943	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
2944	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
2945	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
2946	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
2947	}
2948
2949	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
2950	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
2951	}
2952
2953	/*
2954	* Guest IDTR.
2955	*/
2956	if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_IDTR)
2957	{
2958	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
2959
2960	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt); AssertRC(rc);
2961	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt); AssertRC(rc);
2962
2963	/* Validate. */
2964	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
2965
2966	ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
2967	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
2968	}
2969
2970	return VINF_SUCCESS;
2971	}
2972
2973
2974	/**
2975	* Gets the IEM exception flags for the specified vector and IDT vectoring /
2976	* VM-exit interruption info type.
2977	*
2978	* @returns The IEM exception flags.
2979	* @param uVector The event vector.
2980	* @param uVmxEventType The VMX event type.
2981	*
2982	* @remarks This function currently only constructs flags required for
2983	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
2984	* and CR2 aspects of an exception are not included).
2985	*/
2986	static uint32_t vmxHCGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
2987	{
2988	uint32_t fIemXcptFlags;
2989	switch (uVmxEventType)
2990	{
2991	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
2992	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
2993	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
2994	break;
2995
2996	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
2997	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
2998	break;
2999
3000	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
3001	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
3002	break;
3003
3004	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
3005	{
3006	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
3007	if (uVector == X86_XCPT_BP)
3008	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
3009	else if (uVector == X86_XCPT_OF)
3010	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
3011	else
3012	{
3013	fIemXcptFlags = 0;
3014	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
3015	}
3016	break;
3017	}
3018
3019	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
3020	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
3021	break;
3022
3023	default:
3024	fIemXcptFlags = 0;
3025	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
3026	break;
3027	}
3028	return fIemXcptFlags;
3029	}
3030
3031
3032	/**
3033	* Sets an event as a pending event to be injected into the guest.
3034	*
3035	* @param pVCpu The cross context virtual CPU structure.
3036	* @param u32IntInfo The VM-entry interruption-information field.
3037	* @param cbInstr The VM-entry instruction length in bytes (for
3038	* software interrupts, exceptions and privileged
3039	* software exceptions).
3040	* @param u32ErrCode The VM-entry exception error code.
3041	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
3042	* page-fault.
3043	*/
3044	DECLINLINE(void) vmxHCSetPendingEvent(PVMCPUCC pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
3045	RTGCUINTPTR GCPtrFaultAddress)
3046	{
3047	Assert(!VCPU_2_VMXSTATE(pVCpu).Event.fPending);
3048	VCPU_2_VMXSTATE(pVCpu).Event.fPending = true;
3049	VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo = u32IntInfo;
3050	VCPU_2_VMXSTATE(pVCpu).Event.u32ErrCode = u32ErrCode;
3051	VCPU_2_VMXSTATE(pVCpu).Event.cbInstr = cbInstr;
3052	VCPU_2_VMXSTATE(pVCpu).Event.GCPtrFaultAddress = GCPtrFaultAddress;
3053	}
3054
3055
3056	/**
3057	* Sets an external interrupt as pending-for-injection into the VM.
3058	*
3059	* @param pVCpu The cross context virtual CPU structure.
3060	* @param u8Interrupt The external interrupt vector.
3061	*/
3062	DECLINLINE(void) vmxHCSetPendingExtInt(PVMCPUCC pVCpu, uint8_t u8Interrupt)
3063	{
3064	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
3065	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
3066	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
3067	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
3068	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
3069	}
3070
3071
3072	/**
3073	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
3074	*
3075	* @param pVCpu The cross context virtual CPU structure.
3076	*/
3077	DECLINLINE(void) vmxHCSetPendingXcptNmi(PVMCPUCC pVCpu)
3078	{
3079	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
3080	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
3081	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
3082	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
3083	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
3084	}
3085
3086
3087	/**
3088	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
3089	*
3090	* @param pVCpu The cross context virtual CPU structure.
3091	*/
3092	DECLINLINE(void) vmxHCSetPendingXcptDF(PVMCPUCC pVCpu)
3093	{
3094	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
3095	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
3096	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
3097	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
3098	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
3099	}
3100
3101
3102	/**
3103	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
3104	*
3105	* @param pVCpu The cross context virtual CPU structure.
3106	*/
3107	DECLINLINE(void) vmxHCSetPendingXcptUD(PVMCPUCC pVCpu)
3108	{
3109	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
3110	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
3111	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
3112	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
3113	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
3114	}
3115
3116
3117	/**
3118	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
3119	*
3120	* @param pVCpu The cross context virtual CPU structure.
3121	*/
3122	DECLINLINE(void) vmxHCSetPendingXcptDB(PVMCPUCC pVCpu)
3123	{
3124	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
3125	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
3126	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
3127	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
3128	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
3129	}
3130
3131
3132	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3133	/**
3134	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
3135	*
3136	* @param pVCpu The cross context virtual CPU structure.
3137	* @param u32ErrCode The error code for the general-protection exception.
3138	*/
3139	DECLINLINE(void) vmxHCSetPendingXcptGP(PVMCPUCC pVCpu, uint32_t u32ErrCode)
3140	{
3141	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
3142	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
3143	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
3144	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
3145	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
3146	}
3147
3148
3149	/**
3150	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
3151	*
3152	* @param pVCpu The cross context virtual CPU structure.
3153	* @param u32ErrCode The error code for the stack exception.
3154	*/
3155	DECLINLINE(void) vmxHCSetPendingXcptSS(PVMCPUCC pVCpu, uint32_t u32ErrCode)
3156	{
3157	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
3158	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
3159	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
3160	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
3161	vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
3162	}
3163	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
3164
3165
3166	/**
3167	* Fixes up attributes for the specified segment register.
3168	*
3169	* @param pVCpu The cross context virtual CPU structure.
3170	* @param pSelReg The segment register that needs fixing.
3171	* @param pszRegName The register name (for logging and assertions).
3172	*/
3173	static void vmxHCFixUnusableSegRegAttr(PVMCPUCC pVCpu, PCPUMSELREG pSelReg, const char *pszRegName)
3174	{
3175	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
3176
3177	/*
3178	* If VT-x marks the segment as unusable, most other bits remain undefined:
3179	* - For CS the L, D and G bits have meaning.
3180	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
3181	* - For the remaining data segments no bits are defined.
3182	*
3183	* The present bit and the unusable bit has been observed to be set at the
3184	* same time (the selector was supposed to be invalid as we started executing
3185	* a V8086 interrupt in ring-0).
3186	*
3187	* What should be important for the rest of the VBox code, is that the P bit is
3188	* cleared. Some of the other VBox code recognizes the unusable bit, but
3189	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
3190	* safe side here, we'll strip off P and other bits we don't care about. If
3191	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
3192	*
3193	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
3194	*/
3195	#ifdef VBOX_STRICT
3196	uint32_t const uAttr = pSelReg->Attr.u;
3197	#endif
3198
3199	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
3200	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
3201	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
3202
3203	#ifdef VBOX_STRICT
3204	# ifndef IN_NEM_DARWIN
3205	VMMRZCallRing3Disable(pVCpu);
3206	# endif
3207	Log4Func(("Unusable %s: sel=%#x attr=%#x -> %#x\n", pszRegName, pSelReg->Sel, uAttr, pSelReg->Attr.u));
3208	# ifdef DEBUG_bird
3209	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
3210	("%s: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
3211	pszRegName, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
3212	# endif
3213	# ifndef IN_NEM_DARWIN
3214	VMMRZCallRing3Enable(pVCpu);
3215	# endif
3216	NOREF(uAttr);
3217	#endif
3218	RT_NOREF2(pVCpu, pszRegName);
3219	}
3220
3221
3222	/**
3223	* Imports a guest segment register from the current VMCS into the guest-CPU
3224	* context.
3225	*
3226	* @param pVCpu The cross context virtual CPU structure.
3227	* @param iSegReg The segment register number (X86_SREG_XXX).
3228	*
3229	* @remarks Called with interrupts and/or preemption disabled.
3230	*/
3231	static void vmxHCImportGuestSegReg(PVMCPUCC pVCpu, uint32_t iSegReg)
3232	{
3233	Assert(iSegReg < X86_SREG_COUNT);
3234	Assert((uint32_t)VMX_VMCS16_GUEST_SEG_SEL(iSegReg) == g_aVmcsSegSel[iSegReg]);
3235	Assert((uint32_t)VMX_VMCS32_GUEST_SEG_LIMIT(iSegReg) == g_aVmcsSegLimit[iSegReg]);
3236	Assert((uint32_t)VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(iSegReg) == g_aVmcsSegAttr[iSegReg]);
3237	Assert((uint32_t)VMX_VMCS_GUEST_SEG_BASE(iSegReg) == g_aVmcsSegBase[iSegReg]);
3238
3239	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
3240
3241	uint16_t u16Sel;
3242	int rc = VMX_VMCS_READ_16(pVCpu, VMX_VMCS16_GUEST_SEG_SEL(iSegReg), &u16Sel); AssertRC(rc);
3243	pSelReg->Sel = u16Sel;
3244	pSelReg->ValidSel = u16Sel;
3245
3246	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_SEG_LIMIT(iSegReg), &pSelReg->u32Limit); AssertRC(rc);
3247	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SEG_BASE(iSegReg), &pSelReg->u64Base); AssertRC(rc);
3248
3249	uint32_t u32Attr;
3250	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(iSegReg), &u32Attr); AssertRC(rc);
3251	pSelReg->Attr.u = u32Attr;
3252	if (u32Attr & X86DESCATTR_UNUSABLE)
3253	vmxHCFixUnusableSegRegAttr(pVCpu, pSelReg, "ES\0CS\0SS\0DS\0FS\0GS" + iSegReg * 3);
3254
3255	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
3256	}
3257
3258
3259	/**
3260	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
3261	*
3262	* @param pVCpu The cross context virtual CPU structure.
3263	*
3264	* @remarks Called with interrupts and/or preemption disabled.
3265	*/
3266	static void vmxHCImportGuestLdtr(PVMCPUCC pVCpu)
3267	{
3268	uint16_t u16Sel;
3269	uint64_t u64Base;
3270	uint32_t u32Limit, u32Attr;
3271	int rc = VMX_VMCS_READ_16(pVCpu, VMX_VMCS16_GUEST_LDTR_SEL, &u16Sel); AssertRC(rc);
3272	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit); AssertRC(rc);
3273	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr); AssertRC(rc);
3274	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_LDTR_BASE, &u64Base); AssertRC(rc);
3275
3276	pVCpu->cpum.GstCtx.ldtr.Sel = u16Sel;
3277	pVCpu->cpum.GstCtx.ldtr.ValidSel = u16Sel;
3278	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
3279	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
3280	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
3281	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
3282	if (u32Attr & X86DESCATTR_UNUSABLE)
3283	vmxHCFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, "LDTR");
3284	}
3285
3286
3287	/**
3288	* Imports the guest TR from the current VMCS into the guest-CPU context.
3289	*
3290	* @param pVCpu The cross context virtual CPU structure.
3291	*
3292	* @remarks Called with interrupts and/or preemption disabled.
3293	*/
3294	static void vmxHCImportGuestTr(PVMCPUCC pVCpu)
3295	{
3296	uint16_t u16Sel;
3297	uint64_t u64Base;
3298	uint32_t u32Limit, u32Attr;
3299	int rc = VMX_VMCS_READ_16(pVCpu, VMX_VMCS16_GUEST_TR_SEL, &u16Sel); AssertRC(rc);
3300	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit); AssertRC(rc);
3301	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr); AssertRC(rc);
3302	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_TR_BASE, &u64Base); AssertRC(rc);
3303
3304	pVCpu->cpum.GstCtx.tr.Sel = u16Sel;
3305	pVCpu->cpum.GstCtx.tr.ValidSel = u16Sel;
3306	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
3307	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
3308	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
3309	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
3310	/* TR is the only selector that can never be unusable. */
3311	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
3312	}
3313
3314
3315	/**
3316	* Imports the guest RIP from the VMCS back into the guest-CPU context.
3317	*
3318	* @param pVCpu The cross context virtual CPU structure.
3319	*
3320	* @remarks Called with interrupts and/or preemption disabled, should not assert!
3321	* @remarks Do -not- call this function directly, use vmxHCImportGuestState()
3322	* instead!!!
3323	*/
3324	static void vmxHCImportGuestRip(PVMCPUCC pVCpu)
3325	{
3326	uint64_t u64Val;
3327	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3328	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
3329	{
3330	int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RIP, &u64Val);
3331	AssertRC(rc);
3332
3333	pCtx->rip = u64Val;
3334	EMHistoryUpdatePC(pVCpu, pCtx->rip, false);
3335	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
3336	}
3337	}
3338
3339
3340	/**
3341	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
3342	*
3343	* @param pVCpu The cross context virtual CPU structure.
3344	* @param pVmcsInfo The VMCS info. object.
3345	*
3346	* @remarks Called with interrupts and/or preemption disabled, should not assert!
3347	* @remarks Do -not- call this function directly, use vmxHCImportGuestState()
3348	* instead!!!
3349	*/
3350	static void vmxHCImportGuestRFlags(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
3351	{
3352	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3353	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
3354	{
3355	uint64_t u64Val;
3356	int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RFLAGS, &u64Val);
3357	AssertRC(rc);
3358
3359	pCtx->rflags.u64 = u64Val;
3360	#ifndef IN_NEM_DARWIN
3361	PCVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
3362	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
3363	{
3364	pCtx->eflags.Bits.u1VM = 0;
3365	pCtx->eflags.Bits.u2IOPL = pVmcsInfoShared->RealMode.Eflags.Bits.u2IOPL;
3366	}
3367	#else
3368	RT_NOREF(pVmcsInfo);
3369	#endif
3370	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
3371	}
3372	}
3373
3374
3375	/**
3376	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
3377	* context.
3378	*
3379	* @param pVCpu The cross context virtual CPU structure.
3380	* @param pVmcsInfo The VMCS info. object.
3381	*
3382	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
3383	* do not log!
3384	* @remarks Do -not- call this function directly, use vmxHCImportGuestState()
3385	* instead!!!
3386	*/
3387	static void vmxHCImportGuestIntrState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
3388	{
3389	uint32_t u32Val;
3390	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &u32Val); AssertRC(rc);
3391	if (!u32Val)
3392	{
3393	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3394	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3395	CPUMSetGuestNmiBlocking(pVCpu, false);
3396	}
3397	else
3398	{
3399	/*
3400	* We must import RIP here to set our EM interrupt-inhibited state.
3401	* We also import RFLAGS as our code that evaluates pending interrupts
3402	* before VM-entry requires it.
3403	*/
3404	vmxHCImportGuestRip(pVCpu);
3405	vmxHCImportGuestRFlags(pVCpu, pVmcsInfo);
3406
3407	if (u32Val & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS \| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
3408	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
3409	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3410	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3411
3412	bool const fNmiBlocking = RT_BOOL(u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
3413	CPUMSetGuestNmiBlocking(pVCpu, fNmiBlocking);
3414	}
3415	}
3416
3417
3418	/**
3419	* Worker for VMXR0ImportStateOnDemand.
3420	*
3421	* @returns VBox status code.
3422	* @param pVCpu The cross context virtual CPU structure.
3423	* @param pVmcsInfo The VMCS info. object.
3424	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
3425	*/
3426	static int vmxHCImportGuestState(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
3427	{
3428	int rc = VINF_SUCCESS;
3429	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3430	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3431	uint32_t u32Val;
3432
3433	/*
3434	* Note! This is hack to workaround a mysterious BSOD observed with release builds
3435	* on Windows 10 64-bit hosts. Profile and debug builds are not affected and
3436	* neither are other host platforms.
3437	*
3438	* Committing this temporarily as it prevents BSOD.
3439	*
3440	* Update: This is very likely a compiler optimization bug, see @bugref{9180}.
3441	*/
3442	# ifdef RT_OS_WINDOWS
3443	if (pVM == 0 \|\| pVM == (void *)(uintptr_t)-1)
3444	return VERR_HM_IPE_1;
3445	# endif
3446
3447	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestState, x);
3448
3449	#ifndef IN_NEM_DARWIN
3450	/*
3451	* We disable interrupts to make the updating of the state and in particular
3452	* the fExtrn modification atomic wrt to preemption hooks.
3453	*/
3454	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
3455	#endif
3456
3457	fWhat &= pCtx->fExtrn;
3458	if (fWhat)
3459	{
3460	do
3461	{
3462	if (fWhat & CPUMCTX_EXTRN_RIP)
3463	vmxHCImportGuestRip(pVCpu);
3464
3465	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
3466	vmxHCImportGuestRFlags(pVCpu, pVmcsInfo);
3467
3468	if (fWhat & (CPUMCTX_EXTRN_INHIBIT_INT \| CPUMCTX_EXTRN_INHIBIT_NMI))
3469	vmxHCImportGuestIntrState(pVCpu, pVmcsInfo);
3470
3471	if (fWhat & CPUMCTX_EXTRN_RSP)
3472	{
3473	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RSP, &pCtx->rsp);
3474	AssertRC(rc);
3475	}
3476
3477	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
3478	{
3479	PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
3480	#ifndef IN_NEM_DARWIN
3481	bool const fRealOnV86Active = pVmcsInfoShared->RealMode.fRealOnV86Active;
3482	#else
3483	bool const fRealOnV86Active = false; /* HV supports only unrestricted guest execution. */
3484	#endif
3485	if (fWhat & CPUMCTX_EXTRN_CS)
3486	{
3487	vmxHCImportGuestSegReg(pVCpu, X86_SREG_CS);
3488	vmxHCImportGuestRip(pVCpu);
3489	if (fRealOnV86Active)
3490	pCtx->cs.Attr.u = pVmcsInfoShared->RealMode.AttrCS.u;
3491	EMHistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
3492	}
3493	if (fWhat & CPUMCTX_EXTRN_SS)
3494	{
3495	vmxHCImportGuestSegReg(pVCpu, X86_SREG_SS);
3496	if (fRealOnV86Active)
3497	pCtx->ss.Attr.u = pVmcsInfoShared->RealMode.AttrSS.u;
3498	}
3499	if (fWhat & CPUMCTX_EXTRN_DS)
3500	{
3501	vmxHCImportGuestSegReg(pVCpu, X86_SREG_DS);
3502	if (fRealOnV86Active)
3503	pCtx->ds.Attr.u = pVmcsInfoShared->RealMode.AttrDS.u;
3504	}
3505	if (fWhat & CPUMCTX_EXTRN_ES)
3506	{
3507	vmxHCImportGuestSegReg(pVCpu, X86_SREG_ES);
3508	if (fRealOnV86Active)
3509	pCtx->es.Attr.u = pVmcsInfoShared->RealMode.AttrES.u;
3510	}
3511	if (fWhat & CPUMCTX_EXTRN_FS)
3512	{
3513	vmxHCImportGuestSegReg(pVCpu, X86_SREG_FS);
3514	if (fRealOnV86Active)
3515	pCtx->fs.Attr.u = pVmcsInfoShared->RealMode.AttrFS.u;
3516	}
3517	if (fWhat & CPUMCTX_EXTRN_GS)
3518	{
3519	vmxHCImportGuestSegReg(pVCpu, X86_SREG_GS);
3520	if (fRealOnV86Active)
3521	pCtx->gs.Attr.u = pVmcsInfoShared->RealMode.AttrGS.u;
3522	}
3523	}
3524
3525	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
3526	{
3527	if (fWhat & CPUMCTX_EXTRN_LDTR)
3528	vmxHCImportGuestLdtr(pVCpu);
3529
3530	if (fWhat & CPUMCTX_EXTRN_GDTR)
3531	{
3532	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_GDTR_BASE, &pCtx->gdtr.pGdt); AssertRC(rc);
3533	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRC(rc);
3534	pCtx->gdtr.cbGdt = u32Val;
3535	}
3536
3537	/* Guest IDTR. */
3538	if (fWhat & CPUMCTX_EXTRN_IDTR)
3539	{
3540	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_IDTR_BASE, &pCtx->idtr.pIdt); AssertRC(rc);
3541	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRC(rc);
3542	pCtx->idtr.cbIdt = u32Val;
3543	}
3544
3545	/* Guest TR. */
3546	if (fWhat & CPUMCTX_EXTRN_TR)
3547	{
3548	#ifndef IN_NEM_DARWIN
3549	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
3550	don't need to import that one. */
3551	if (!pVmcsInfo->pShared->RealMode.fRealOnV86Active)
3552	#endif
3553	vmxHCImportGuestTr(pVCpu);
3554	}
3555	}
3556
3557	if (fWhat & CPUMCTX_EXTRN_DR7)
3558	{
3559	#ifndef IN_NEM_DARWIN
3560	if (!pVCpu->hmr0.s.fUsingHyperDR7)
3561	#endif
3562	{
3563	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_DR7, &pCtx->dr[7]);
3564	AssertRC(rc);
3565	}
3566	}
3567
3568	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
3569	{
3570	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip); AssertRC(rc);
3571	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp); AssertRC(rc);
3572	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRC(rc);
3573	pCtx->SysEnter.cs = u32Val;
3574	}
3575
3576	#ifndef IN_NEM_DARWIN
3577	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
3578	{
3579	if ( pVM->hmr0.s.fAllow64BitGuests
3580	&& (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
3581	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
3582	}
3583
3584	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
3585	{
3586	if ( pVM->hmr0.s.fAllow64BitGuests
3587	&& (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
3588	{
3589	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
3590	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
3591	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
3592	}
3593	}
3594
3595	if (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
3596	{
3597	PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
3598	PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
3599	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
3600	Assert(pMsrs);
3601	Assert(cMsrs <= VMX_MISC_MAX_MSRS(g_HmMsrs.u.vmx.u64Misc));
3602	Assert(sizeof(pMsrs) cMsrs <= X86_PAGE_4K_SIZE);
3603	for (uint32_t i = 0; i < cMsrs; i++)
3604	{
3605	uint32_t const idMsr = pMsrs[i].u32Msr;
3606	switch (idMsr)
3607	{
3608	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break;
3609	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break;
3610	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
3611	default:
3612	{
3613	uint32_t idxLbrMsr;
3614	if (VM_IS_VMX_LBR(pVM))
3615	{
3616	if (hmR0VmxIsLbrBranchFromMsr(pVM, idMsr, &idxLbrMsr))
3617	{
3618	Assert(idxLbrMsr < RT_ELEMENTS(pVmcsInfoShared->au64LbrFromIpMsr));
3619	pVmcsInfoShared->au64LbrFromIpMsr[idxLbrMsr] = pMsrs[i].u64Value;
3620	break;
3621	}
3622	if (hmR0VmxIsLbrBranchToMsr(pVM, idMsr, &idxLbrMsr))
3623	{
3624	Assert(idxLbrMsr < RT_ELEMENTS(pVmcsInfoShared->au64LbrFromIpMsr));
3625	pVmcsInfoShared->au64LbrToIpMsr[idxLbrMsr] = pMsrs[i].u64Value;
3626	break;
3627	}
3628	if (idMsr == pVM->hmr0.s.vmx.idLbrTosMsr)
3629	{
3630	pVmcsInfoShared->u64LbrTosMsr = pMsrs[i].u64Value;
3631	break;
3632	}
3633	/* Fallthru (no break) */
3634	}
3635	pCtx->fExtrn = 0;
3636	VCPU_2_VMXSTATE(pVCpu).u32HMError = pMsrs->u32Msr;
3637	ASMSetFlags(fEFlags);
3638	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
3639	return VERR_HM_UNEXPECTED_LD_ST_MSR;
3640	}
3641	}
3642	}
3643	}
3644	#endif
3645
3646	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
3647	{
3648	if (fWhat & CPUMCTX_EXTRN_CR0)
3649	{
3650	uint64_t u64Cr0;
3651	uint64_t u64Shadow;
3652	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR0, &u64Cr0); AssertRC(rc);
3653	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow); AssertRC(rc);
3654	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
3655	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
3656	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
3657	#else
3658	if (!CPUMIsGuestInVmxNonRootMode(pCtx))
3659	{
3660	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
3661	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
3662	}
3663	else
3664	{
3665	/*
3666	* We've merged the guest and nested-guest's CR0 guest/host mask while executing
3667	* the nested-guest using hardware-assisted VMX. Accordingly we need to
3668	* re-construct CR0. See @bugref{9180#c95} for details.
3669	*/
3670	PCVMXVMCSINFO const pVmcsInfoGst = &pVCpu->hmr0.s.vmx.VmcsInfo;
3671	PVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
3672	u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask)
3673	\| (pVmcsNstGst->u64GuestCr0.u & pVmcsNstGst->u64Cr0Mask.u)
3674	\| (u64Shadow & (pVmcsInfoGst->u64Cr0Mask & ~pVmcsNstGst->u64Cr0Mask.u));
3675	}
3676	#endif
3677	#ifndef IN_NEM_DARWIN
3678	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
3679	#endif
3680	CPUMSetGuestCR0(pVCpu, u64Cr0);
3681	#ifndef IN_NEM_DARWIN
3682	VMMRZCallRing3Enable(pVCpu);
3683	#endif
3684	}
3685
3686	if (fWhat & CPUMCTX_EXTRN_CR4)
3687	{
3688	uint64_t u64Cr4;
3689	uint64_t u64Shadow;
3690	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR4, &u64Cr4); AssertRC(rc);
3691	rc \|= VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow); AssertRC(rc);
3692	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
3693	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
3694	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
3695	#else
3696	if (!CPUMIsGuestInVmxNonRootMode(pCtx))
3697	{
3698	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
3699	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
3700	}
3701	else
3702	{
3703	/*
3704	* We've merged the guest and nested-guest's CR4 guest/host mask while executing
3705	* the nested-guest using hardware-assisted VMX. Accordingly we need to
3706	* re-construct CR4. See @bugref{9180#c95} for details.
3707	*/
3708	PCVMXVMCSINFO const pVmcsInfoGst = &pVCpu->hmr0.s.vmx.VmcsInfo;
3709	PVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
3710	u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask)
3711	\| (pVmcsNstGst->u64GuestCr4.u & pVmcsNstGst->u64Cr4Mask.u)
3712	\| (u64Shadow & (pVmcsInfoGst->u64Cr4Mask & ~pVmcsNstGst->u64Cr4Mask.u));
3713	}
3714	#endif
3715	pCtx->cr4 = u64Cr4;
3716	}
3717
3718	if (fWhat & CPUMCTX_EXTRN_CR3)
3719	{
3720	/* CR0.PG bit changes are always intercepted, so it's up to date. */
3721	if ( VM_IS_VMX_UNRESTRICTED_GUEST(pVM)
3722	\|\| ( VM_IS_VMX_NESTED_PAGING(pVM)
3723	&& CPUMIsGuestPagingEnabledEx(pCtx)))
3724	{
3725	uint64_t u64Cr3;
3726	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR3, &u64Cr3); AssertRC(rc);
3727	if (pCtx->cr3 != u64Cr3)
3728	{
3729	pCtx->cr3 = u64Cr3;
3730	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
3731	}
3732
3733	/*
3734	* If the guest is in PAE mode, sync back the PDPE's into the guest state.
3735	* CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date.
3736	*/
3737	if (CPUMIsGuestInPAEModeEx(pCtx))
3738	{
3739	X86PDPE aPaePdpes[4];
3740	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE0_FULL, &aPaePdpes[0].u); AssertRC(rc);
3741	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE1_FULL, &aPaePdpes[1].u); AssertRC(rc);
3742	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE2_FULL, &aPaePdpes[2].u); AssertRC(rc);
3743	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE3_FULL, &aPaePdpes[3].u); AssertRC(rc);
3744	if (memcmp(&aPaePdpes[0], &pCtx->aPaePdpes[0], sizeof(aPaePdpes)))
3745	{
3746	memcpy(&pCtx->aPaePdpes[0], &aPaePdpes[0], sizeof(aPaePdpes));
3747	/* PGM now updates PAE PDPTEs while updating CR3. */
3748	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
3749	}
3750	}
3751	}
3752	}
3753	}
3754
3755	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3756	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
3757	{
3758	if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
3759	&& !CPUMIsGuestInVmxNonRootMode(pCtx))
3760	{
3761	Assert(CPUMIsGuestInVmxRootMode(pCtx));
3762	rc = vmxHCCopyShadowToNstGstVmcs(pVCpu, pVmcsInfo);
3763	if (RT_SUCCESS(rc))
3764	{ /* likely */ }
3765	else
3766	break;
3767	}
3768	}
3769	#endif
3770	} while (0);
3771
3772	if (RT_SUCCESS(rc))
3773	{
3774	/* Update fExtrn. */
3775	pCtx->fExtrn &= ~fWhat;
3776
3777	/* If everything has been imported, clear the HM keeper bit. */
3778	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
3779	{
3780	#ifndef IN_NEM_DARWIN
3781	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
3782	#else
3783	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_NEM;
3784	#endif
3785	Assert(!pCtx->fExtrn);
3786	}
3787	}
3788	}
3789	#ifndef IN_NEM_DARWIN
3790	else
3791	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
3792
3793	/*
3794	* Restore interrupts.
3795	*/
3796	ASMSetFlags(fEFlags);
3797	#endif
3798
3799	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestState, x);
3800
3801	if (RT_SUCCESS(rc))
3802	{ /* likely */ }
3803	else
3804	return rc;
3805
3806	/*
3807	* Honor any pending CR3 updates.
3808	*
3809	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> VMXR0CallRing3Callback()
3810	* -> VMMRZCallRing3Disable() -> vmxHCImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
3811	* -> continue with VM-exit handling -> vmxHCImportGuestState() and here we are.
3812	*
3813	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
3814	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
3815	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
3816	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
3817	*
3818	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
3819	*
3820	* The force-flag is checked first as it's cheaper for potential superfluous calls to this function.
3821	*/
3822	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3)
3823	#ifndef IN_NEM_DARWIN
3824	&& VMMRZCallRing3IsEnabled(pVCpu)
3825	#endif
3826	)
3827	{
3828	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
3829	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
3830	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
3831	}
3832
3833	return VINF_SUCCESS;
3834	}
3835
3836
3837	/**
3838	* Check per-VM and per-VCPU force flag actions that require us to go back to
3839	* ring-3 for one reason or another.
3840	*
3841	* @returns Strict VBox status code (i.e. informational status codes too)
3842	* @retval VINF_SUCCESS if we don't have any actions that require going back to
3843	* ring-3.
3844	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
3845	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
3846	* interrupts)
3847	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
3848	* all EMTs to be in ring-3.
3849	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
3850	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
3851	* to the EM loop.
3852	*
3853	* @param pVCpu The cross context virtual CPU structure.
3854	* @param fIsNestedGuest Flag whether this is for a for a pending nested guest event.
3855	* @param fStepping Whether we are single-stepping the guest using the
3856	* hypervisor debugger.
3857	*
3858	* @remarks This might cause nested-guest VM-exits, caller must check if the guest
3859	* is no longer in VMX non-root mode.
3860	*/
3861	static VBOXSTRICTRC vmxHCCheckForceFlags(PVMCPUCC pVCpu, bool fIsNestedGuest, bool fStepping)
3862	{
3863	#ifndef IN_NEM_DARWIN
3864	Assert(VMMRZCallRing3IsEnabled(pVCpu));
3865	#endif
3866
3867	/*
3868	* Update pending interrupts into the APIC's IRR.
3869	*/
3870	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
3871	APICUpdatePendingInterrupts(pVCpu);
3872
3873	/*
3874	* Anything pending? Should be more likely than not if we're doing a good job.
3875	*/
3876	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3877	if ( !fStepping
3878	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
3879	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
3880	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
3881	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
3882	return VINF_SUCCESS;
3883
3884	/* Pending PGM C3 sync. */
3885	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
3886	{
3887	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3888	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
3889	VBOXSTRICTRC rcStrict = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
3890	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
3891	if (rcStrict != VINF_SUCCESS)
3892	{
3893	AssertRC(VBOXSTRICTRC_VAL(rcStrict));
3894	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
3895	return rcStrict;
3896	}
3897	}
3898
3899	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
3900	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
3901	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
3902	{
3903	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchHmToR3FF);
3904	int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
3905	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
3906	return rc;
3907	}
3908
3909	/* Pending VM request packets, such as hardware interrupts. */
3910	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
3911	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
3912	{
3913	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchVmReq);
3914	Log4Func(("Pending VM request forcing us back to ring-3\n"));
3915	return VINF_EM_PENDING_REQUEST;
3916	}
3917
3918	/* Pending PGM pool flushes. */
3919	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
3920	{
3921	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchPgmPoolFlush);
3922	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
3923	return VINF_PGM_POOL_FLUSH_PENDING;
3924	}
3925
3926	/* Pending DMA requests. */
3927	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
3928	{
3929	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchDma);
3930	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
3931	return VINF_EM_RAW_TO_R3;
3932	}
3933
3934	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3935	/*
3936	* Pending nested-guest events.
3937	*
3938	* Please note the priority of these events are specified and important.
3939	* See Intel spec. 29.4.3.2 "APIC-Write Emulation".
3940	* See Intel spec. 6.9 "Priority Among Simultaneous Exceptions And Interrupts".
3941	*/
3942	if (fIsNestedGuest)
3943	{
3944	/* Pending nested-guest APIC-write. */
3945	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
3946	{
3947	Log4Func(("Pending nested-guest APIC-write\n"));
3948	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitApicWrite(pVCpu);
3949	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
3950	return rcStrict;
3951	}
3952
3953	/* Pending nested-guest monitor-trap flag (MTF). */
3954	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_MTF))
3955	{
3956	Log4Func(("Pending nested-guest MTF\n"));
3957	VBOXSTRICTRC rcStrict = IEMExecVmxVmexit(pVCpu, VMX_EXIT_MTF, 0 /* uExitQual */);
3958	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
3959	return rcStrict;
3960	}
3961
3962	/* Pending nested-guest VMX-preemption timer expired. */
3963	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER))
3964	{
3965	Log4Func(("Pending nested-guest preempt timer\n"));
3966	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitPreemptTimer(pVCpu);
3967	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
3968	return rcStrict;
3969	}
3970	}
3971	#else
3972	NOREF(fIsNestedGuest);
3973	#endif
3974
3975	return VINF_SUCCESS;
3976	}
3977
3978
3979	/**
3980	* Converts any TRPM trap into a pending HM event. This is typically used when
3981	* entering from ring-3 (not longjmp returns).
3982	*
3983	* @param pVCpu The cross context virtual CPU structure.
3984	*/
3985	static void vmxHCTrpmTrapToPendingEvent(PVMCPUCC pVCpu)
3986	{
3987	Assert(TRPMHasTrap(pVCpu));
3988	Assert(!VCPU_2_VMXSTATE(pVCpu).Event.fPending);
3989
3990	uint8_t uVector;
3991	TRPMEVENT enmTrpmEvent;
3992	uint32_t uErrCode;
3993	RTGCUINTPTR GCPtrFaultAddress;
3994	uint8_t cbInstr;
3995	bool fIcebp;
3996
3997	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr, &fIcebp);
3998	AssertRC(rc);
3999
4000	uint32_t u32IntInfo;
4001	u32IntInfo = uVector \| VMX_IDT_VECTORING_INFO_VALID;
4002	u32IntInfo \|= HMTrpmEventTypeToVmxEventType(uVector, enmTrpmEvent, fIcebp);
4003
4004	rc = TRPMResetTrap(pVCpu);
4005	AssertRC(rc);
4006	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
4007	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
4008
4009	vmxHCSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
4010	}
4011
4012
4013	/**
4014	* Converts the pending HM event into a TRPM trap.
4015	*
4016	* @param pVCpu The cross context virtual CPU structure.
4017	*/
4018	static void vmxHCPendingEventToTrpmTrap(PVMCPUCC pVCpu)
4019	{
4020	Assert(VCPU_2_VMXSTATE(pVCpu).Event.fPending);
4021
4022	/* If a trap was already pending, we did something wrong! */
4023	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
4024
4025	uint32_t const u32IntInfo = VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo;
4026	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(u32IntInfo);
4027	TRPMEVENT const enmTrapType = HMVmxEventTypeToTrpmEventType(u32IntInfo);
4028
4029	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
4030
4031	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
4032	AssertRC(rc);
4033
4034	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
4035	TRPMSetErrorCode(pVCpu, VCPU_2_VMXSTATE(pVCpu).Event.u32ErrCode);
4036
4037	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(u32IntInfo))
4038	TRPMSetFaultAddress(pVCpu, VCPU_2_VMXSTATE(pVCpu).Event.GCPtrFaultAddress);
4039	else
4040	{
4041	uint8_t const uVectorType = VMX_IDT_VECTORING_INFO_TYPE(u32IntInfo);
4042	switch (uVectorType)
4043	{
4044	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
4045	TRPMSetTrapDueToIcebp(pVCpu);
4046	RT_FALL_THRU();
4047	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
4048	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
4049	{
4050	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4051	\|\| ( uVector == X86_XCPT_BP /* INT3 */
4052	\|\| uVector == X86_XCPT_OF /* INTO */
4053	\|\| uVector == X86_XCPT_DB /* INT1 (ICEBP) */),
4054	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
4055	TRPMSetInstrLength(pVCpu, VCPU_2_VMXSTATE(pVCpu).Event.cbInstr);
4056	break;
4057	}
4058	}
4059	}
4060
4061	/* We're now done converting the pending event. */
4062	VCPU_2_VMXSTATE(pVCpu).Event.fPending = false;
4063	}
4064
4065
4066	/**
4067	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
4068	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
4069	*
4070	* @param pVCpu The cross context virtual CPU structure.
4071	* @param pVmcsInfo The VMCS info. object.
4072	*/
4073	static void vmxHCSetIntWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
4074	{
4075	if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
4076	{
4077	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
4078	{
4079	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
4080	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
4081	AssertRC(rc);
4082	}
4083	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
4084	}
4085
4086
4087	/**
4088	* Clears the interrupt-window exiting control in the VMCS.
4089	*
4090	* @param pVCpu The cross context virtual CPU structure.
4091	* @param pVmcsInfo The VMCS info. object.
4092	*/
4093	DECLINLINE(void) vmxHCClearIntWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
4094	{
4095	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
4096	{
4097	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
4098	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
4099	AssertRC(rc);
4100	}
4101	}
4102
4103
4104	/**
4105	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
4106	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
4107	*
4108	* @param pVCpu The cross context virtual CPU structure.
4109	* @param pVmcsInfo The VMCS info. object.
4110	*/
4111	static void vmxHCSetNmiWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
4112	{
4113	if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
4114	{
4115	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
4116	{
4117	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
4118	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
4119	AssertRC(rc);
4120	Log4Func(("Setup NMI-window exiting\n"));
4121	}
4122	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
4123	}
4124
4125
4126	/**
4127	* Clears the NMI-window exiting control in the VMCS.
4128	*
4129	* @param pVCpu The cross context virtual CPU structure.
4130	* @param pVmcsInfo The VMCS info. object.
4131	*/
4132	DECLINLINE(void) vmxHCClearNmiWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
4133	{
4134	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
4135	{
4136	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
4137	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
4138	AssertRC(rc);
4139	}
4140	}
4141
4142
4143	/**
4144	* Injects an event into the guest upon VM-entry by updating the relevant fields
4145	* in the VM-entry area in the VMCS.
4146	*
4147	* @returns Strict VBox status code (i.e. informational status codes too).
4148	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
4149	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
4150	*
4151	* @param pVCpu The cross context virtual CPU structure.
4152	* @param pVmxTransient The VMX-transient structure.
4153	* @param pEvent The event being injected.
4154	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state. This
4155	* will be updated if necessary. This cannot not be NULL.
4156	* @param fStepping Whether we're single-stepping guest execution and should
4157	* return VINF_EM_DBG_STEPPED if the event is injected
4158	* directly (registers modified by us, not by hardware on
4159	* VM-entry).
4160	*/
4161	static VBOXSTRICTRC vmxHCInjectEventVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNestedGuest, PCHMEVENT pEvent, bool fStepping,
4162	uint32_t *pfIntrState)
4163	{
4164	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
4165	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
4166	Assert(pfIntrState);
4167
4168	#ifdef IN_NEM_DARWIN
4169	RT_NOREF(fIsNestedGuest, fStepping, pfIntrState);
4170	#endif
4171
4172	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4173	uint32_t u32IntInfo = pEvent->u64IntInfo;
4174	uint32_t const u32ErrCode = pEvent->u32ErrCode;
4175	uint32_t const cbInstr = pEvent->cbInstr;
4176	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
4177	uint8_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
4178	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
4179
4180	#ifdef VBOX_STRICT
4181	/*
4182	* Validate the error-code-valid bit for hardware exceptions.
4183	* No error codes for exceptions in real-mode.
4184	*
4185	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
4186	*/
4187	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
4188	&& !CPUMIsGuestInRealModeEx(pCtx))
4189	{
4190	switch (uVector)
4191	{
4192	case X86_XCPT_PF:
4193	case X86_XCPT_DF:
4194	case X86_XCPT_TS:
4195	case X86_XCPT_NP:
4196	case X86_XCPT_SS:
4197	case X86_XCPT_GP:
4198	case X86_XCPT_AC:
4199	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
4200	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
4201	RT_FALL_THRU();
4202	default:
4203	break;
4204	}
4205	}
4206
4207	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
4208	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
4209	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
4210	#endif
4211
4212	RT_NOREF(uVector);
4213	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
4214	\|\| uIntType == VMX_EXIT_INT_INFO_TYPE_NMI
4215	\|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT
4216	\|\| uIntType == VMX_EXIT_INT_INFO_TYPE_SW_XCPT)
4217	{
4218	Assert(uVector <= X86_XCPT_LAST);
4219	Assert(uIntType != VMX_EXIT_INT_INFO_TYPE_NMI \|\| uVector == X86_XCPT_NMI);
4220	Assert(uIntType != VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT \|\| uVector == X86_XCPT_DB);
4221	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).aStatInjectedXcpts[uVector]);
4222	}
4223	else
4224	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).aStatInjectedIrqs[uVector & MASK_INJECT_IRQ_STAT]);
4225
4226	/*
4227	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
4228	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
4229	* interrupt handler in the (real-mode) guest.
4230	*
4231	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
4232	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
4233	*/
4234	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
4235	{
4236	#ifndef IN_NEM_DARWIN
4237	if (pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fUnrestrictedGuest)
4238	#endif
4239	{
4240	/*
4241	* For CPUs with unrestricted guest execution enabled and with the guest
4242	* in real-mode, we must not set the deliver-error-code bit.
4243	*
4244	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
4245	*/
4246	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
4247	}
4248	#ifndef IN_NEM_DARWIN
4249	else
4250	{
4251	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4252	Assert(PDMVmmDevHeapIsEnabled(pVM));
4253	Assert(pVM->hm.s.vmx.pRealModeTSS);
4254	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
4255
4256	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
4257	int rc2 = vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
4258	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
4259	AssertRCReturn(rc2, rc2);
4260
4261	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
4262	size_t const cbIdtEntry = sizeof(X86IDTR16);
4263	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
4264	{
4265	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
4266	if (uVector == X86_XCPT_DF)
4267	return VINF_EM_RESET;
4268
4269	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
4270	No error codes for exceptions in real-mode. */
4271	if (uVector == X86_XCPT_GP)
4272	{
4273	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
4274	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
4275	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
4276	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
4277	HMEVENT EventXcptDf;
4278	RT_ZERO(EventXcptDf);
4279	EventXcptDf.u64IntInfo = uXcptDfInfo;
4280	return vmxHCInjectEventVmcs(pVCpu, pVmcsInfo, fIsNestedGuest, &EventXcptDf, fStepping, pfIntrState);
4281	}
4282
4283	/*
4284	* If we're injecting an event with no valid IDT entry, inject a #GP.
4285	* No error codes for exceptions in real-mode.
4286	*
4287	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
4288	*/
4289	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
4290	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
4291	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
4292	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
4293	HMEVENT EventXcptGp;
4294	RT_ZERO(EventXcptGp);
4295	EventXcptGp.u64IntInfo = uXcptGpInfo;
4296	return vmxHCInjectEventVmcs(pVCpu, pVmcsInfo, fIsNestedGuest, &EventXcptGp, fStepping, pfIntrState);
4297	}
4298
4299	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
4300	uint16_t uGuestIp = pCtx->ip;
4301	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
4302	{
4303	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
4304	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
4305	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
4306	}
4307	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
4308	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
4309
4310	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
4311	X86IDTR16 IdtEntry;
4312	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
4313	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
4314	AssertRCReturn(rc2, rc2);
4315
4316	/* Construct the stack frame for the interrupt/exception handler. */
4317	VBOXSTRICTRC rcStrict;
4318	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
4319	if (rcStrict == VINF_SUCCESS)
4320	{
4321	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
4322	if (rcStrict == VINF_SUCCESS)
4323	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
4324	}
4325
4326	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
4327	if (rcStrict == VINF_SUCCESS)
4328	{
4329	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
4330	pCtx->rip = IdtEntry.offSel;
4331	pCtx->cs.Sel = IdtEntry.uSel;
4332	pCtx->cs.ValidSel = IdtEntry.uSel;
4333	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
4334	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
4335	&& uVector == X86_XCPT_PF)
4336	pCtx->cr2 = GCPtrFault;
4337
4338	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
4339	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
4340	\| HM_CHANGED_GUEST_RSP);
4341
4342	/*
4343	* If we delivered a hardware exception (other than an NMI) and if there was
4344	* block-by-STI in effect, we should clear it.
4345	*/
4346	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
4347	{
4348	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
4349	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
4350	Log4Func(("Clearing inhibition due to STI\n"));
4351	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
4352	}
4353
4354	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
4355	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
4356
4357	/*
4358	* The event has been truly dispatched to the guest. Mark it as no longer pending so
4359	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
4360	*/
4361	VCPU_2_VMXSTATE(pVCpu).Event.fPending = false;
4362
4363	/*
4364	* If we eventually support nested-guest execution without unrestricted guest execution,
4365	* we should set fInterceptEvents here.
4366	*/
4367	Assert(!fIsNestedGuest);
4368
4369	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
4370	if (fStepping)
4371	rcStrict = VINF_EM_DBG_STEPPED;
4372	}
4373	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
4374	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
4375	return rcStrict;
4376	}
4377	#else
4378	RT_NOREF(pVmcsInfo);
4379	#endif
4380	}
4381
4382	/*
4383	* Validate.
4384	*/
4385	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
4386	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
4387
4388	/*
4389	* Inject the event into the VMCS.
4390	*/
4391	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
4392	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
4393	rc \|= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
4394	rc \|= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
4395	AssertRC(rc);
4396
4397	/*
4398	* Update guest CR2 if this is a page-fault.
4399	*/
4400	if (VMX_ENTRY_INT_INFO_IS_XCPT_PF(u32IntInfo))
4401	pCtx->cr2 = GCPtrFault;
4402
4403	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
4404	return VINF_SUCCESS;
4405	}
4406
4407
4408	/**
4409	* Evaluates the event to be delivered to the guest and sets it as the pending
4410	* event.
4411	*
4412	* Toggling of interrupt force-flags here is safe since we update TRPM on premature
4413	* exits to ring-3 before executing guest code, see vmxHCExitToRing3(). We must
4414	* NOT restore these force-flags.
4415	*
4416	* @returns Strict VBox status code (i.e. informational status codes too).
4417	* @param pVCpu The cross context virtual CPU structure.
4418	* @param pVmcsInfo The VMCS information structure.
4419	* @param fIsNestedGuest Flag whether the evaluation happens for a nestd guest.
4420	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
4421	*/
4422	static VBOXSTRICTRC vmxHCEvaluatePendingEvent(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNestedGuest, uint32_t *pfIntrState)
4423	{
4424	Assert(pfIntrState);
4425	Assert(!TRPMHasTrap(pVCpu));
4426
4427	/*
4428	* Compute/update guest-interruptibility state related FFs.
4429	* The FFs will be used below while evaluating events to be injected.
4430	*/
4431	*pfIntrState = vmxHCGetGuestIntrStateAndUpdateFFs(pVCpu);
4432
4433	/*
4434	* Evaluate if a new event needs to be injected.
4435	* An event that's already pending has already performed all necessary checks.
4436	*/
4437	if ( !VCPU_2_VMXSTATE(pVCpu).Event.fPending
4438	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4439	{
4440	/** @todo SMI. SMIs take priority over NMIs. */
4441
4442	/*
4443	* NMIs.
4444	* NMIs take priority over external interrupts.
4445	*/
4446	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4447	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4448	#endif
4449	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
4450	{
4451	/*
4452	* For a guest, the FF always indicates the guest's ability to receive an NMI.
4453	*
4454	* For a nested-guest, the FF always indicates the outer guest's ability to
4455	* receive an NMI while the guest-interruptibility state bit depends on whether
4456	* the nested-hypervisor is using virtual-NMIs.
4457	*/
4458	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
4459	{
4460	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4461	if ( fIsNestedGuest
4462	&& CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_NMI_EXIT))
4463	return IEMExecVmxVmexitXcptNmi(pVCpu);
4464	#endif
4465	vmxHCSetPendingXcptNmi(pVCpu);
4466	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
4467	Log4Func(("NMI pending injection\n"));
4468
4469	/* We've injected the NMI, bail. */
4470	return VINF_SUCCESS;
4471	}
4472	else if (!fIsNestedGuest)
4473	vmxHCSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
4474	}
4475
4476	/*
4477	* External interrupts (PIC/APIC).
4478	* Once PDMGetInterrupt() returns a valid interrupt we -must- deliver it.
4479	* We cannot re-request the interrupt from the controller again.
4480	*/
4481	if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
4482	&& !VCPU_2_VMXSTATE(pVCpu).fSingleInstruction)
4483	{
4484	Assert(!DBGFIsStepping(pVCpu));
4485	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
4486	AssertRC(rc);
4487
4488	/*
4489	* We must not check EFLAGS directly when executing a nested-guest, use
4490	* CPUMIsGuestPhysIntrEnabled() instead as EFLAGS.IF does not control the blocking of
4491	* external interrupts when "External interrupt exiting" is set. This fixes a nasty
4492	* SMP hang while executing nested-guest VCPUs on spinlocks which aren't rescued by
4493	* other VM-exits (like a preemption timer), see @bugref{9562#c18}.
4494	*
4495	* See Intel spec. 25.4.1 "Event Blocking".
4496	*/
4497	if (CPUMIsGuestPhysIntrEnabled(pVCpu))
4498	{
4499	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4500	if ( fIsNestedGuest
4501	&& CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
4502	{
4503	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0 /* uVector /, true / fIntPending */);
4504	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
4505	return rcStrict;
4506	}
4507	#endif
4508	uint8_t u8Interrupt;
4509	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
4510	if (RT_SUCCESS(rc))
4511	{
4512	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4513	if ( fIsNestedGuest
4514	&& CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
4515	{
4516	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
4517	Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
4518	return rcStrict;
4519	}
4520	#endif
4521	vmxHCSetPendingExtInt(pVCpu, u8Interrupt);
4522	Log4Func(("External interrupt (%#x) pending injection\n", u8Interrupt));
4523	}
4524	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
4525	{
4526	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchTprMaskedIrq);
4527
4528	if ( !fIsNestedGuest
4529	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
4530	vmxHCApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
4531	/* else: for nested-guests, TPR threshold is picked up while merging VMCS controls. */
4532
4533	/*
4534	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
4535	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
4536	* need to re-set this force-flag here.
4537	*/
4538	}
4539	else
4540	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchGuestIrq);
4541
4542	/* We've injected the interrupt or taken necessary action, bail. */
4543	return VINF_SUCCESS;
4544	}
4545	if (!fIsNestedGuest)
4546	vmxHCSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
4547	}
4548	}
4549	else if (!fIsNestedGuest)
4550	{
4551	/*
4552	* An event is being injected or we are in an interrupt shadow. Check if another event is
4553	* pending. If so, instruct VT-x to cause a VM-exit as soon as the guest is ready to accept
4554	* the pending event.
4555	*/
4556	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
4557	vmxHCSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
4558	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
4559	&& !VCPU_2_VMXSTATE(pVCpu).fSingleInstruction)
4560	vmxHCSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
4561	}
4562	/* else: for nested-guests, NMI/interrupt-window exiting will be picked up when merging VMCS controls. */
4563
4564	return VINF_SUCCESS;
4565	}
4566
4567
4568	/**
4569	* Injects any pending events into the guest if the guest is in a state to
4570	* receive them.
4571	*
4572	* @returns Strict VBox status code (i.e. informational status codes too).
4573	* @param pVCpu The cross context virtual CPU structure.
4574	* @param fIsNestedGuest Flag whether the event injection happens for a nested guest.
4575	* @param fIntrState The VT-x guest-interruptibility state.
4576	* @param fStepping Whether we are single-stepping the guest using the
4577	* hypervisor debugger and should return
4578	* VINF_EM_DBG_STEPPED if the event was dispatched
4579	* directly.
4580	*/
4581	static VBOXSTRICTRC vmxHCInjectPendingEvent(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNestedGuest, uint32_t fIntrState, bool fStepping)
4582	{
4583	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
4584	#ifndef IN_NEM_DARWIN
4585	Assert(VMMRZCallRing3IsEnabled(pVCpu));
4586	#endif
4587
4588	#ifdef VBOX_STRICT
4589	/*
4590	* Verify guest-interruptibility state.
4591	*
4592	* We put this in a scoped block so we do not accidentally use fBlockSti or fBlockMovSS,
4593	* since injecting an event may modify the interruptibility state and we must thus always
4594	* use fIntrState.
4595	*/
4596	{
4597	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
4598	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
4599	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
4600	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
4601	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
4602	Assert(!TRPMHasTrap(pVCpu));
4603	NOREF(fBlockMovSS); NOREF(fBlockSti);
4604	}
4605	#endif
4606
4607	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
4608	if (VCPU_2_VMXSTATE(pVCpu).Event.fPending)
4609	{
4610	/*
4611	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
4612	* pending even while injecting an event and in this case, we want a VM-exit as soon as
4613	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
4614	*
4615	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
4616	*/
4617	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo);
4618	#ifdef VBOX_STRICT
4619	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
4620	{
4621	Assert(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
4622	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI));
4623	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
4624	}
4625	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
4626	{
4627	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI));
4628	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI));
4629	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
4630	}
4631	#endif
4632	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo,
4633	uIntType));
4634
4635	/*
4636	* Inject the event and get any changes to the guest-interruptibility state.
4637	*
4638	* The guest-interruptibility state may need to be updated if we inject the event
4639	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
4640	*/
4641	rcStrict = vmxHCInjectEventVmcs(pVCpu, pVmcsInfo, fIsNestedGuest, &VCPU_2_VMXSTATE(pVCpu).Event, fStepping, &fIntrState);
4642	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
4643
4644	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
4645	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectInterrupt);
4646	else
4647	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectXcpt);
4648	}
4649
4650	/*
4651	* Deliver any pending debug exceptions if the guest is single-stepping using EFLAGS.TF and
4652	* is an interrupt shadow (block-by-STI or block-by-MOV SS).
4653	*/
4654	if ( (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
4655	&& !fIsNestedGuest)
4656	{
4657	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
4658
4659	if (!VCPU_2_VMXSTATE(pVCpu).fSingleInstruction)
4660	{
4661	/*
4662	* Set or clear the BS bit depending on whether the trap flag is active or not. We need
4663	* to do both since we clear the BS bit from the VMCS while exiting to ring-3.
4664	*/
4665	Assert(!DBGFIsStepping(pVCpu));
4666	uint8_t const fTrapFlag = !!(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_TF);
4667	int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, fTrapFlag << VMX_BF_VMCS_PENDING_DBG_XCPT_BS_SHIFT);
4668	AssertRC(rc);
4669	}
4670	else
4671	{
4672	/*
4673	* We must not deliver a debug exception when single-stepping over STI/Mov-SS in the
4674	* hypervisor debugger using EFLAGS.TF but rather clear interrupt inhibition. However,
4675	* we take care of this case in vmxHCExportSharedDebugState and also the case if
4676	* we use MTF, so just make sure it's called before executing guest-code.
4677	*/
4678	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_DR_MASK);
4679	}
4680	}
4681	/* else: for nested-guest currently handling while merging controls. */
4682
4683	/*
4684	* Finally, update the guest-interruptibility state.
4685	*
4686	* This is required for the real-on-v86 software interrupt injection, for
4687	* pending debug exceptions as well as updates to the guest state from ring-3 (IEM).
4688	*/
4689	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, fIntrState);
4690	AssertRC(rc);
4691
4692	/*
4693	* There's no need to clear the VM-entry interruption-information field here if we're not
4694	* injecting anything. VT-x clears the valid bit on every VM-exit.
4695	*
4696	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
4697	*/
4698
4699	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
4700	return rcStrict;
4701	}
4702
4703
4704	/**
4705	* Tries to determine what part of the guest-state VT-x has deemed as invalid
4706	* and update error record fields accordingly.
4707	*
4708	* @returns VMX_IGS_* error codes.
4709	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
4710	* wrong with the guest state.
4711	*
4712	* @param pVCpu The cross context virtual CPU structure.
4713	* @param pVmcsInfo The VMCS info. object.
4714	*
4715	* @remarks This function assumes our cache of the VMCS controls
4716	* are valid, i.e. vmxHCCheckCachedVmcsCtls() succeeded.
4717	*/
4718	static uint32_t vmxHCCheckGuestState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
4719	{
4720	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
4721	#define HMVMX_CHECK_BREAK(expr, err) do { \
4722	if (!(expr)) { uError = (err); break; } \
4723	} while (0)
4724
4725	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4726	uint32_t uError = VMX_IGS_ERROR;
4727	uint32_t u32IntrState = 0;
4728	#ifndef IN_NEM_DARWIN
4729	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4730	bool const fUnrestrictedGuest = VM_IS_VMX_UNRESTRICTED_GUEST(pVM);
4731	#else
4732	bool const fUnrestrictedGuest = true;
4733	#endif
4734	do
4735	{
4736	int rc;
4737
4738	/*
4739	* Guest-interruptibility state.
4740	*
4741	* Read this first so that any check that fails prior to those that actually
4742	* require the guest-interruptibility state would still reflect the correct
4743	* VMCS value and avoids causing further confusion.
4744	*/
4745	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
4746	AssertRC(rc);
4747
4748	uint32_t u32Val;
4749	uint64_t u64Val;
4750
4751	/*
4752	* CR0.
4753	*/
4754	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
4755	uint64_t fSetCr0 = (g_HmMsrs.u.vmx.u64Cr0Fixed0 & g_HmMsrs.u.vmx.u64Cr0Fixed1);
4756	uint64_t const fZapCr0 = (g_HmMsrs.u.vmx.u64Cr0Fixed0 \| g_HmMsrs.u.vmx.u64Cr0Fixed1);
4757	/* Exceptions for unrestricted guest execution for CR0 fixed bits (PE, PG).
4758	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
4759	if (fUnrestrictedGuest)
4760	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
4761
4762	uint64_t u64GuestCr0;
4763	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR0, &u64GuestCr0);
4764	AssertRC(rc);
4765	HMVMX_CHECK_BREAK((u64GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
4766	HMVMX_CHECK_BREAK(!(u64GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
4767	if ( !fUnrestrictedGuest
4768	&& (u64GuestCr0 & X86_CR0_PG)
4769	&& !(u64GuestCr0 & X86_CR0_PE))
4770	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
4771
4772	/*
4773	* CR4.
4774	*/
4775	/** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
4776	uint64_t const fSetCr4 = (g_HmMsrs.u.vmx.u64Cr4Fixed0 & g_HmMsrs.u.vmx.u64Cr4Fixed1);
4777	uint64_t const fZapCr4 = (g_HmMsrs.u.vmx.u64Cr4Fixed0 \| g_HmMsrs.u.vmx.u64Cr4Fixed1);
4778
4779	uint64_t u64GuestCr4;
4780	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR4, &u64GuestCr4);
4781	AssertRC(rc);
4782	HMVMX_CHECK_BREAK((u64GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
4783	HMVMX_CHECK_BREAK(!(u64GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
4784
4785	/*
4786	* IA32_DEBUGCTL MSR.
4787	*/
4788	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
4789	AssertRC(rc);
4790	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
4791	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
4792	{
4793	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
4794	}
4795	uint64_t u64DebugCtlMsr = u64Val;
4796
4797	#ifdef VBOX_STRICT
4798	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY, &u32Val);
4799	AssertRC(rc);
4800	Assert(u32Val == pVmcsInfo->u32EntryCtls);
4801	#endif
4802	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
4803
4804	/*
4805	* RIP and RFLAGS.
4806	*/
4807	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RIP, &u64Val);
4808	AssertRC(rc);
4809	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
4810	if ( !fLongModeGuest
4811	\|\| !pCtx->cs.Attr.n.u1Long)
4812	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
4813	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
4814	* must be identical if the "IA-32e mode guest" VM-entry
4815	* control is 1 and CS.L is 1. No check applies if the
4816	* CPU supports 64 linear-address bits. */
4817
4818	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
4819	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RFLAGS, &u64Val);
4820	AssertRC(rc);
4821	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
4822	VMX_IGS_RFLAGS_RESERVED);
4823	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
4824	uint32_t const u32Eflags = u64Val;
4825
4826	if ( fLongModeGuest
4827	\|\| ( fUnrestrictedGuest
4828	&& !(u64GuestCr0 & X86_CR0_PE)))
4829	{
4830	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
4831	}
4832
4833	uint32_t u32EntryInfo;
4834	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
4835	AssertRC(rc);
4836	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
4837	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
4838
4839	/*
4840	* 64-bit checks.
4841	*/
4842	if (fLongModeGuest)
4843	{
4844	HMVMX_CHECK_BREAK(u64GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
4845	HMVMX_CHECK_BREAK(u64GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
4846	}
4847
4848	if ( !fLongModeGuest
4849	&& (u64GuestCr4 & X86_CR4_PCIDE))
4850	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
4851
4852	/** @todo CR3 field must be such that bits 63:52 and bits in the range
4853	* 51:32 beyond the processor's physical-address width are 0. */
4854
4855	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
4856	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
4857	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
4858
4859	#ifndef IN_NEM_DARWIN
4860	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
4861	AssertRC(rc);
4862	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
4863
4864	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
4865	AssertRC(rc);
4866	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
4867	#endif
4868
4869	/*
4870	* PERF_GLOBAL MSR.
4871	*/
4872	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
4873	{
4874	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
4875	AssertRC(rc);
4876	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
4877	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
4878	}
4879
4880	/*
4881	* PAT MSR.
4882	*/
4883	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
4884	{
4885	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
4886	AssertRC(rc);
4887	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
4888	for (unsigned i = 0; i < 8; i++)
4889	{
4890	uint8_t u8Val = (u64Val & 0xff);
4891	if ( u8Val != 0 /* UC */
4892	&& u8Val != 1 /* WC */
4893	&& u8Val != 4 /* WT */
4894	&& u8Val != 5 /* WP */
4895	&& u8Val != 6 /* WB */
4896	&& u8Val != 7 /* UC- */)
4897	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
4898	u64Val >>= 8;
4899	}
4900	}
4901
4902	/*
4903	* EFER MSR.
4904	*/
4905	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
4906	{
4907	Assert(g_fHmVmxSupportsVmcsEfer);
4908	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
4909	AssertRC(rc);
4910	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
4911	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
4912	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
4913	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
4914	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
4915	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
4916	* iemVmxVmentryCheckGuestState(). */
4917	HMVMX_CHECK_BREAK( fUnrestrictedGuest
4918	\|\| !(u64GuestCr0 & X86_CR0_PG)
4919	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
4920	VMX_IGS_EFER_LMA_LME_MISMATCH);
4921	}
4922
4923	/*
4924	* Segment registers.
4925	*/
4926	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
4927	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
4928	if (!(u32Eflags & X86_EFL_VM))
4929	{
4930	/* CS */
4931	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
4932	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
4933	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
4934	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4935	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
4936	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
4937	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
4938	/* CS cannot be loaded with NULL in protected mode. */
4939	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
4940	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
4941	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
4942	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
4943	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
4944	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
4945	else if (fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
4946	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
4947	else
4948	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
4949
4950	/* SS */
4951	HMVMX_CHECK_BREAK( fUnrestrictedGuest
4952	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
4953	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
4954	if ( !(pCtx->cr0 & X86_CR0_PE)
4955	\|\| pCtx->cs.Attr.n.u4Type == 3)
4956	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
4957
4958	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4959	{
4960	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
4961	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
4962	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
4963	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
4964	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4965	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
4966	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
4967	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
4968	}
4969
4970	/* DS, ES, FS, GS - only check for usable selectors, see vmxHCExportGuestSReg(). */
4971	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4972	{
4973	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
4974	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
4975	HMVMX_CHECK_BREAK( fUnrestrictedGuest
4976	\|\| pCtx->ds.Attr.n.u4Type > 11
4977	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
4978	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
4979	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
4980	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4981	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
4982	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
4983	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
4984	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4985	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
4986	}
4987	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4988	{
4989	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
4990	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
4991	HMVMX_CHECK_BREAK( fUnrestrictedGuest
4992	\|\| pCtx->es.Attr.n.u4Type > 11
4993	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
4994	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
4995	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
4996	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
4997	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
4998	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
4999	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
5000	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5001	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
5002	}
5003	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
5004	{
5005	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
5006	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
5007	HMVMX_CHECK_BREAK( fUnrestrictedGuest
5008	\|\| pCtx->fs.Attr.n.u4Type > 11
5009	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
5010	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
5011	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
5012	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
5013	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
5014	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
5015	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
5016	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5017	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
5018	}
5019	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
5020	{
5021	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
5022	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
5023	HMVMX_CHECK_BREAK( fUnrestrictedGuest
5024	\|\| pCtx->gs.Attr.n.u4Type > 11
5025	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
5026	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
5027	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
5028	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
5029	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
5030	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
5031	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
5032	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5033	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
5034	}
5035	/* 64-bit capable CPUs. */
5036	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
5037	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
5038	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
5039	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
5040	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
5041	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
5042	VMX_IGS_LONGMODE_SS_BASE_INVALID);
5043	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
5044	VMX_IGS_LONGMODE_DS_BASE_INVALID);
5045	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
5046	VMX_IGS_LONGMODE_ES_BASE_INVALID);
5047	}
5048	else
5049	{
5050	/* V86 mode checks. */
5051	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
5052	if (pVmcsInfo->pShared->RealMode.fRealOnV86Active)
5053	{
5054	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
5055	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
5056	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
5057	}
5058	else
5059	{
5060	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
5061	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
5062	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
5063	}
5064
5065	/* CS */
5066	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
5067	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
5068	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
5069	/* SS */
5070	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
5071	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
5072	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
5073	/* DS */
5074	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
5075	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
5076	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
5077	/* ES */
5078	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
5079	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
5080	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
5081	/* FS */
5082	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
5083	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
5084	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
5085	/* GS */
5086	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
5087	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
5088	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
5089	/* 64-bit capable CPUs. */
5090	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
5091	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
5092	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
5093	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
5094	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
5095	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
5096	VMX_IGS_LONGMODE_SS_BASE_INVALID);
5097	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
5098	VMX_IGS_LONGMODE_DS_BASE_INVALID);
5099	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
5100	VMX_IGS_LONGMODE_ES_BASE_INVALID);
5101	}
5102
5103	/*
5104	* TR.
5105	*/
5106	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
5107	/* 64-bit capable CPUs. */
5108	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
5109	if (fLongModeGuest)
5110	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
5111	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
5112	else
5113	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
5114	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
5115	VMX_IGS_TR_ATTR_TYPE_INVALID);
5116	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
5117	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
5118	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
5119	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
5120	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
5121	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
5122	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
5123	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
5124
5125	/*
5126	* GDTR and IDTR (64-bit capable checks).
5127	*/
5128	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
5129	AssertRC(rc);
5130	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
5131
5132	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
5133	AssertRC(rc);
5134	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
5135
5136	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
5137	AssertRC(rc);
5138	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
5139
5140	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
5141	AssertRC(rc);
5142	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
5143
5144	/*
5145	* Guest Non-Register State.
5146	*/
5147	/* Activity State. */
5148	uint32_t u32ActivityState;
5149	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
5150	AssertRC(rc);
5151	HMVMX_CHECK_BREAK( !u32ActivityState
5152	\|\| (u32ActivityState & RT_BF_GET(g_HmMsrs.u.vmx.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
5153	VMX_IGS_ACTIVITY_STATE_INVALID);
5154	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
5155	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
5156
5157	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
5158	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
5159	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
5160
5161	/** @todo Activity state and injecting interrupts. Left as a todo since we
5162	* currently don't use activity states but ACTIVE. */
5163
5164	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
5165	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
5166
5167	/* Guest interruptibility-state. */
5168	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
5169	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
5170	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
5171	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
5172	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
5173	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
5174	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
5175	if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
5176	{
5177	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
5178	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
5179	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
5180	}
5181	else if (VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
5182	{
5183	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
5184	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
5185	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
5186	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
5187	}
5188	/** @todo Assumes the processor is not in SMM. */
5189	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
5190	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
5191	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
5192	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
5193	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
5194	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
5195	&& VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
5196	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI), VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
5197
5198	/* Pending debug exceptions. */
5199	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
5200	AssertRC(rc);
5201	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
5202	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
5203	u32Val = u64Val; /* For pending debug exceptions checks below. */
5204
5205	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
5206	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
5207	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
5208	{
5209	if ( (u32Eflags & X86_EFL_TF)
5210	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
5211	{
5212	/* Bit 14 is PendingDebug.BS. */
5213	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
5214	}
5215	if ( !(u32Eflags & X86_EFL_TF)
5216	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
5217	{
5218	/* Bit 14 is PendingDebug.BS. */
5219	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
5220	}
5221	}
5222
5223	#ifndef IN_NEM_DARWIN
5224	/* VMCS link pointer. */
5225	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
5226	AssertRC(rc);
5227	if (u64Val != UINT64_C(0xffffffffffffffff))
5228	{
5229	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
5230	/** @todo Bits beyond the processor's physical-address width MBZ. */
5231	/** @todo SMM checks. */
5232	Assert(pVmcsInfo->HCPhysShadowVmcs == u64Val);
5233	Assert(pVmcsInfo->pvShadowVmcs);
5234	VMXVMCSREVID VmcsRevId;
5235	VmcsRevId.u = (uint32_t )pVmcsInfo->pvShadowVmcs;
5236	HMVMX_CHECK_BREAK(VmcsRevId.n.u31RevisionId == RT_BF_GET(g_HmMsrs.u.vmx.u64Basic, VMX_BF_BASIC_VMCS_ID),
5237	VMX_IGS_VMCS_LINK_PTR_SHADOW_VMCS_ID_INVALID);
5238	HMVMX_CHECK_BREAK(VmcsRevId.n.fIsShadowVmcs == (uint32_t)!!(pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING),
5239	VMX_IGS_VMCS_LINK_PTR_NOT_SHADOW);
5240	}
5241
5242	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
5243	* not using nested paging? */
5244	if ( VM_IS_VMX_NESTED_PAGING(pVM)
5245	&& !fLongModeGuest
5246	&& CPUMIsGuestInPAEModeEx(pCtx))
5247	{
5248	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
5249	AssertRC(rc);
5250	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
5251
5252	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
5253	AssertRC(rc);
5254	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
5255
5256	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
5257	AssertRC(rc);
5258	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
5259
5260	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
5261	AssertRC(rc);
5262	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
5263	}
5264	#endif
5265
5266	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
5267	if (uError == VMX_IGS_ERROR)
5268	uError = VMX_IGS_REASON_NOT_FOUND;
5269	} while (0);
5270
5271	VCPU_2_VMXSTATE(pVCpu).u32HMError = uError;
5272	VCPU_2_VMXSTATE(pVCpu).vmx.LastError.u32GuestIntrState = u32IntrState;
5273	return uError;
5274
5275	#undef HMVMX_ERROR_BREAK
5276	#undef HMVMX_CHECK_BREAK
5277	}
5278	/** @} */
5279
5280
5281	#ifndef HMVMX_USE_FUNCTION_TABLE
5282	/**
5283	* Handles a guest VM-exit from hardware-assisted VMX execution.
5284	*
5285	* @returns Strict VBox status code (i.e. informational status codes too).
5286	* @param pVCpu The cross context virtual CPU structure.
5287	* @param pVmxTransient The VMX-transient structure.
5288	*/
5289	DECLINLINE(VBOXSTRICTRC) vmxHCHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5290	{
5291	#ifdef DEBUG_ramshankar
5292	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
5293	do { \
5294	if (a_fSave != 0) \
5295	vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL); \
5296	VBOXSTRICTRC rcStrict = a_CallExpr; \
5297	if (a_fSave != 0) \
5298	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST); \
5299	return rcStrict; \
5300	} while (0)
5301	#else
5302	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
5303	#endif
5304	uint32_t const uExitReason = pVmxTransient->uExitReason;
5305	switch (uExitReason)
5306	{
5307	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, vmxHCExitEptMisconfig(pVCpu, pVmxTransient));
5308	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, vmxHCExitEptViolation(pVCpu, pVmxTransient));
5309	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, vmxHCExitIoInstr(pVCpu, pVmxTransient));
5310	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, vmxHCExitCpuid(pVCpu, pVmxTransient));
5311	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, vmxHCExitRdtsc(pVCpu, pVmxTransient));
5312	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, vmxHCExitRdtscp(pVCpu, pVmxTransient));
5313	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, vmxHCExitApicAccess(pVCpu, pVmxTransient));
5314	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, vmxHCExitXcptOrNmi(pVCpu, pVmxTransient));
5315	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, vmxHCExitMovCRx(pVCpu, pVmxTransient));
5316	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, vmxHCExitExtInt(pVCpu, pVmxTransient));
5317	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, vmxHCExitIntWindow(pVCpu, pVmxTransient));
5318	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, vmxHCExitTprBelowThreshold(pVCpu, pVmxTransient));
5319	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, vmxHCExitMwait(pVCpu, pVmxTransient));
5320	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, vmxHCExitMonitor(pVCpu, pVmxTransient));
5321	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, vmxHCExitTaskSwitch(pVCpu, pVmxTransient));
5322	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, vmxHCExitPreemptTimer(pVCpu, pVmxTransient));
5323	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, vmxHCExitRdmsr(pVCpu, pVmxTransient));
5324	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, vmxHCExitWrmsr(pVCpu, pVmxTransient));
5325	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, vmxHCExitVmcall(pVCpu, pVmxTransient));
5326	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, vmxHCExitMovDRx(pVCpu, pVmxTransient));
5327	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, vmxHCExitHlt(pVCpu, pVmxTransient));
5328	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, vmxHCExitInvd(pVCpu, pVmxTransient));
5329	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, vmxHCExitInvlpg(pVCpu, pVmxTransient));
5330	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, vmxHCExitMtf(pVCpu, pVmxTransient));
5331	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, vmxHCExitPause(pVCpu, pVmxTransient));
5332	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, vmxHCExitWbinvd(pVCpu, pVmxTransient));
5333	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, vmxHCExitXsetbv(pVCpu, pVmxTransient));
5334	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, vmxHCExitInvpcid(pVCpu, pVmxTransient));
5335	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, vmxHCExitGetsec(pVCpu, pVmxTransient));
5336	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, vmxHCExitRdpmc(pVCpu, pVmxTransient));
5337	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5338	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, vmxHCExitVmclear(pVCpu, pVmxTransient));
5339	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, vmxHCExitVmlaunch(pVCpu, pVmxTransient));
5340	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, vmxHCExitVmptrld(pVCpu, pVmxTransient));
5341	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, vmxHCExitVmptrst(pVCpu, pVmxTransient));
5342	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, vmxHCExitVmread(pVCpu, pVmxTransient));
5343	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, vmxHCExitVmwrite(pVCpu, pVmxTransient));
5344	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, vmxHCExitVmresume(pVCpu, pVmxTransient));
5345	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, vmxHCExitVmxoff(pVCpu, pVmxTransient));
5346	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, vmxHCExitVmxon(pVCpu, pVmxTransient));
5347	case VMX_EXIT_INVVPID: VMEXIT_CALL_RET(0, vmxHCExitInvvpid(pVCpu, pVmxTransient));
5348	#else
5349	case VMX_EXIT_VMCLEAR:
5350	case VMX_EXIT_VMLAUNCH:
5351	case VMX_EXIT_VMPTRLD:
5352	case VMX_EXIT_VMPTRST:
5353	case VMX_EXIT_VMREAD:
5354	case VMX_EXIT_VMRESUME:
5355	case VMX_EXIT_VMWRITE:
5356	case VMX_EXIT_VMXOFF:
5357	case VMX_EXIT_VMXON:
5358	case VMX_EXIT_INVVPID:
5359	return vmxHCExitSetPendingXcptUD(pVCpu, pVmxTransient);
5360	#endif
5361	#if defined(VBOX_WITH_NESTED_HWVIRT_VMX) && defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT)
5362	case VMX_EXIT_INVEPT: VMEXIT_CALL_RET(0, vmxHCExitInvept(pVCpu, pVmxTransient));
5363	#else
5364	case VMX_EXIT_INVEPT: return vmxHCExitSetPendingXcptUD(pVCpu, pVmxTransient);
5365	#endif
5366
5367	case VMX_EXIT_TRIPLE_FAULT: return vmxHCExitTripleFault(pVCpu, pVmxTransient);
5368	case VMX_EXIT_NMI_WINDOW: return vmxHCExitNmiWindow(pVCpu, pVmxTransient);
5369	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return vmxHCExitErrInvalidGuestState(pVCpu, pVmxTransient);
5370
5371	case VMX_EXIT_INIT_SIGNAL:
5372	case VMX_EXIT_SIPI:
5373	case VMX_EXIT_IO_SMI:
5374	case VMX_EXIT_SMI:
5375	case VMX_EXIT_ERR_MSR_LOAD:
5376	case VMX_EXIT_ERR_MACHINE_CHECK:
5377	case VMX_EXIT_PML_FULL:
5378	case VMX_EXIT_VIRTUALIZED_EOI:
5379	case VMX_EXIT_GDTR_IDTR_ACCESS:
5380	case VMX_EXIT_LDTR_TR_ACCESS:
5381	case VMX_EXIT_APIC_WRITE:
5382	case VMX_EXIT_RDRAND:
5383	case VMX_EXIT_RSM:
5384	case VMX_EXIT_VMFUNC:
5385	case VMX_EXIT_ENCLS:
5386	case VMX_EXIT_RDSEED:
5387	case VMX_EXIT_XSAVES:
5388	case VMX_EXIT_XRSTORS:
5389	case VMX_EXIT_UMWAIT:
5390	case VMX_EXIT_TPAUSE:
5391	case VMX_EXIT_LOADIWKEY:
5392	default:
5393	return vmxHCExitErrUnexpected(pVCpu, pVmxTransient);
5394	}
5395	#undef VMEXIT_CALL_RET
5396	}
5397	#endif /* !HMVMX_USE_FUNCTION_TABLE */
5398
5399
5400	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5401	/**
5402	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
5403	*
5404	* @returns Strict VBox status code (i.e. informational status codes too).
5405	* @param pVCpu The cross context virtual CPU structure.
5406	* @param pVmxTransient The VMX-transient structure.
5407	*/
5408	DECLINLINE(VBOXSTRICTRC) vmxHCHandleExitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5409	{
5410	uint32_t const uExitReason = pVmxTransient->uExitReason;
5411	switch (uExitReason)
5412	{
5413	case VMX_EXIT_EPT_MISCONFIG: return vmxHCExitEptMisconfig(pVCpu, pVmxTransient);
5414	case VMX_EXIT_EPT_VIOLATION: return vmxHCExitEptViolation(pVCpu, pVmxTransient);
5415	case VMX_EXIT_XCPT_OR_NMI: return vmxHCExitXcptOrNmiNested(pVCpu, pVmxTransient);
5416	case VMX_EXIT_IO_INSTR: return vmxHCExitIoInstrNested(pVCpu, pVmxTransient);
5417	case VMX_EXIT_HLT: return vmxHCExitHltNested(pVCpu, pVmxTransient);
5418
5419	/*
5420	* We shouldn't direct host physical interrupts to the nested-guest.
5421	*/
5422	case VMX_EXIT_EXT_INT:
5423	return vmxHCExitExtInt(pVCpu, pVmxTransient);
5424
5425	/*
5426	* Instructions that cause VM-exits unconditionally or the condition is
5427	* always is taken solely from the nested hypervisor (meaning if the VM-exit
5428	* happens, it's guaranteed to be a nested-guest VM-exit).
5429	*
5430	* - Provides VM-exit instruction length ONLY.
5431	*/
5432	case VMX_EXIT_CPUID: /* Unconditional. */
5433	case VMX_EXIT_VMCALL:
5434	case VMX_EXIT_GETSEC:
5435	case VMX_EXIT_INVD:
5436	case VMX_EXIT_XSETBV:
5437	case VMX_EXIT_VMLAUNCH:
5438	case VMX_EXIT_VMRESUME:
5439	case VMX_EXIT_VMXOFF:
5440	case VMX_EXIT_ENCLS: /* Condition specified solely by nested hypervisor. */
5441	case VMX_EXIT_VMFUNC:
5442	return vmxHCExitInstrNested(pVCpu, pVmxTransient);
5443
5444	/*
5445	* Instructions that cause VM-exits unconditionally or the condition is
5446	* always is taken solely from the nested hypervisor (meaning if the VM-exit
5447	* happens, it's guaranteed to be a nested-guest VM-exit).
5448	*
5449	* - Provides VM-exit instruction length.
5450	* - Provides VM-exit information.
5451	* - Optionally provides Exit qualification.
5452	*
5453	* Since Exit qualification is 0 for all VM-exits where it is not
5454	* applicable, reading and passing it to the guest should produce
5455	* defined behavior.
5456	*
5457	* See Intel spec. 27.2.1 "Basic VM-Exit Information".
5458	*/
5459	case VMX_EXIT_INVEPT: /* Unconditional. */
5460	case VMX_EXIT_INVVPID:
5461	case VMX_EXIT_VMCLEAR:
5462	case VMX_EXIT_VMPTRLD:
5463	case VMX_EXIT_VMPTRST:
5464	case VMX_EXIT_VMXON:
5465	case VMX_EXIT_GDTR_IDTR_ACCESS: /* Condition specified solely by nested hypervisor. */
5466	case VMX_EXIT_LDTR_TR_ACCESS:
5467	case VMX_EXIT_RDRAND:
5468	case VMX_EXIT_RDSEED:
5469	case VMX_EXIT_XSAVES:
5470	case VMX_EXIT_XRSTORS:
5471	case VMX_EXIT_UMWAIT:
5472	case VMX_EXIT_TPAUSE:
5473	return vmxHCExitInstrWithInfoNested(pVCpu, pVmxTransient);
5474
5475	case VMX_EXIT_RDTSC: return vmxHCExitRdtscNested(pVCpu, pVmxTransient);
5476	case VMX_EXIT_RDTSCP: return vmxHCExitRdtscpNested(pVCpu, pVmxTransient);
5477	case VMX_EXIT_RDMSR: return vmxHCExitRdmsrNested(pVCpu, pVmxTransient);
5478	case VMX_EXIT_WRMSR: return vmxHCExitWrmsrNested(pVCpu, pVmxTransient);
5479	case VMX_EXIT_INVLPG: return vmxHCExitInvlpgNested(pVCpu, pVmxTransient);
5480	case VMX_EXIT_INVPCID: return vmxHCExitInvpcidNested(pVCpu, pVmxTransient);
5481	case VMX_EXIT_TASK_SWITCH: return vmxHCExitTaskSwitchNested(pVCpu, pVmxTransient);
5482	case VMX_EXIT_WBINVD: return vmxHCExitWbinvdNested(pVCpu, pVmxTransient);
5483	case VMX_EXIT_MTF: return vmxHCExitMtfNested(pVCpu, pVmxTransient);
5484	case VMX_EXIT_APIC_ACCESS: return vmxHCExitApicAccessNested(pVCpu, pVmxTransient);
5485	case VMX_EXIT_APIC_WRITE: return vmxHCExitApicWriteNested(pVCpu, pVmxTransient);
5486	case VMX_EXIT_VIRTUALIZED_EOI: return vmxHCExitVirtEoiNested(pVCpu, pVmxTransient);
5487	case VMX_EXIT_MOV_CRX: return vmxHCExitMovCRxNested(pVCpu, pVmxTransient);
5488	case VMX_EXIT_INT_WINDOW: return vmxHCExitIntWindowNested(pVCpu, pVmxTransient);
5489	case VMX_EXIT_NMI_WINDOW: return vmxHCExitNmiWindowNested(pVCpu, pVmxTransient);
5490	case VMX_EXIT_TPR_BELOW_THRESHOLD: return vmxHCExitTprBelowThresholdNested(pVCpu, pVmxTransient);
5491	case VMX_EXIT_MWAIT: return vmxHCExitMwaitNested(pVCpu, pVmxTransient);
5492	case VMX_EXIT_MONITOR: return vmxHCExitMonitorNested(pVCpu, pVmxTransient);
5493	case VMX_EXIT_PAUSE: return vmxHCExitPauseNested(pVCpu, pVmxTransient);
5494
5495	case VMX_EXIT_PREEMPT_TIMER:
5496	{
5497	/** @todo NSTVMX: Preempt timer. */
5498	return vmxHCExitPreemptTimer(pVCpu, pVmxTransient);
5499	}
5500
5501	case VMX_EXIT_MOV_DRX: return vmxHCExitMovDRxNested(pVCpu, pVmxTransient);
5502	case VMX_EXIT_RDPMC: return vmxHCExitRdpmcNested(pVCpu, pVmxTransient);
5503
5504	case VMX_EXIT_VMREAD:
5505	case VMX_EXIT_VMWRITE: return vmxHCExitVmreadVmwriteNested(pVCpu, pVmxTransient);
5506
5507	case VMX_EXIT_TRIPLE_FAULT: return vmxHCExitTripleFaultNested(pVCpu, pVmxTransient);
5508	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return vmxHCExitErrInvalidGuestStateNested(pVCpu, pVmxTransient);
5509
5510	case VMX_EXIT_INIT_SIGNAL:
5511	case VMX_EXIT_SIPI:
5512	case VMX_EXIT_IO_SMI:
5513	case VMX_EXIT_SMI:
5514	case VMX_EXIT_ERR_MSR_LOAD:
5515	case VMX_EXIT_ERR_MACHINE_CHECK:
5516	case VMX_EXIT_PML_FULL:
5517	case VMX_EXIT_RSM:
5518	default:
5519	return vmxHCExitErrUnexpected(pVCpu, pVmxTransient);
5520	}
5521	}
5522	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
5523
5524
5525	/** @name VM-exit helpers.
5526	* @{
5527	*/
5528	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
5529	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit helpers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
5530	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
5531
5532	/** Macro for VM-exits called unexpectedly. */
5533	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_HmError) \
5534	do { \
5535	VCPU_2_VMXSTATE((a_pVCpu)).u32HMError = (a_HmError); \
5536	return VERR_VMX_UNEXPECTED_EXIT; \
5537	} while (0)
5538
5539	#ifdef VBOX_STRICT
5540	# ifndef IN_NEM_DARWIN
5541	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
5542	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
5543	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
5544
5545	# define HMVMX_ASSERT_PREEMPT_CPUID() \
5546	do { \
5547	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
5548	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
5549	} while (0)
5550
5551	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
5552	do { \
5553	AssertPtr((a_pVCpu)); \
5554	AssertPtr((a_pVmxTransient)); \
5555	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
5556	Assert((a_pVmxTransient)->pVmcsInfo); \
5557	Assert(ASMIntAreEnabled()); \
5558	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
5559	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
5560	Log4Func(("vcpu[%RU32]\n", (a_pVCpu)->idCpu)); \
5561	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
5562	if (!VMMRZCallRing3IsEnabled((a_pVCpu))) \
5563	HMVMX_ASSERT_PREEMPT_CPUID(); \
5564	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
5565	} while (0)
5566	# else
5567	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() do { } while(0)
5568	# define HMVMX_ASSERT_PREEMPT_CPUID() do { } while(0)
5569	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
5570	do { \
5571	AssertPtr((a_pVCpu)); \
5572	AssertPtr((a_pVmxTransient)); \
5573	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
5574	Assert((a_pVmxTransient)->pVmcsInfo); \
5575	Log4Func(("vcpu[%RU32]\n", (a_pVCpu)->idCpu)); \
5576	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
5577	} while (0)
5578	# endif
5579
5580	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
5581	do { \
5582	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); \
5583	Assert((a_pVmxTransient)->fIsNestedGuest); \
5584	} while (0)
5585
5586	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
5587	do { \
5588	Log4Func(("\n")); \
5589	} while (0)
5590	#else
5591	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
5592	do { \
5593	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
5594	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
5595	} while (0)
5596
5597	# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
5598	do { HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); } while (0)
5599
5600	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
5601	#endif
5602
5603	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5604	/** Macro that does the necessary privilege checks and intercepted VM-exits for
5605	* guests that attempted to execute a VMX instruction. */
5606	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
5607	do \
5608	{ \
5609	VBOXSTRICTRC rcStrictTmp = vmxHCCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
5610	if (rcStrictTmp == VINF_SUCCESS) \
5611	{ /* likely */ } \
5612	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
5613	{ \
5614	Assert((a_pVCpu)->hm.s.Event.fPending); \
5615	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
5616	return VINF_SUCCESS; \
5617	} \
5618	else \
5619	{ \
5620	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
5621	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
5622	} \
5623	} while (0)
5624
5625	/** Macro that decodes a memory operand for an VM-exit caused by an instruction. */
5626	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
5627	do \
5628	{ \
5629	VBOXSTRICTRC rcStrictTmp = vmxHCDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
5630	(a_pGCPtrEffAddr)); \
5631	if (rcStrictTmp == VINF_SUCCESS) \
5632	{ /* likely */ } \
5633	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
5634	{ \
5635	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
5636	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
5637	NOREF(uXcptTmp); \
5638	return VINF_SUCCESS; \
5639	} \
5640	else \
5641	{ \
5642	Log4Func(("vmxHCDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
5643	return rcStrictTmp; \
5644	} \
5645	} while (0)
5646	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
5647
5648
5649	/**
5650	* Advances the guest RIP by the specified number of bytes.
5651	*
5652	* @param pVCpu The cross context virtual CPU structure.
5653	* @param cbInstr Number of bytes to advance the RIP by.
5654	*
5655	* @remarks No-long-jump zone!!!
5656	*/
5657	DECLINLINE(void) vmxHCAdvanceGuestRipBy(PVMCPUCC pVCpu, uint32_t cbInstr)
5658	{
5659	/* Advance the RIP. */
5660	pVCpu->cpum.GstCtx.rip += cbInstr;
5661	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP);
5662
5663	/* Update interrupt inhibition. */
5664	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
5665	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
5666	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5667	}
5668
5669
5670	/**
5671	* Advances the guest RIP after reading it from the VMCS.
5672	*
5673	* @returns VBox status code, no informational status codes.
5674	* @param pVCpu The cross context virtual CPU structure.
5675	* @param pVmxTransient The VMX-transient structure.
5676	*
5677	* @remarks No-long-jump zone!!!
5678	*/
5679	static int vmxHCAdvanceGuestRip(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5680	{
5681	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
5682	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
5683	AssertRCReturn(rc, rc);
5684
5685	vmxHCAdvanceGuestRipBy(pVCpu, pVmxTransient->cbExitInstr);
5686	return VINF_SUCCESS;
5687	}
5688
5689
5690	/**
5691	* Handle a condition that occurred while delivering an event through the guest or
5692	* nested-guest IDT.
5693	*
5694	* @returns Strict VBox status code (i.e. informational status codes too).
5695	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5696	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5697	* to continue execution of the guest which will delivery the \#DF.
5698	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5699	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5700	*
5701	* @param pVCpu The cross context virtual CPU structure.
5702	* @param pVmxTransient The VMX-transient structure.
5703	*
5704	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
5705	* Additionally, HMVMX_READ_EXIT_QUALIFICATION is required if the VM-exit
5706	* is due to an EPT violation, PML full or SPP-related event.
5707	*
5708	* @remarks No-long-jump zone!!!
5709	*/
5710	static VBOXSTRICTRC vmxHCCheckExitDueToEventDelivery(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
5711	{
5712	Assert(!VCPU_2_VMXSTATE(pVCpu).Event.fPending);
5713	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
5714	if ( pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
5715	\|\| pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
5716	\|\| pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
5717	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_EXIT_QUALIFICATION);
5718
5719	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5720	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5721	uint32_t const uIdtVectorInfo = pVmxTransient->uIdtVectoringInfo;
5722	uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
5723	if (VMX_IDT_VECTORING_INFO_IS_VALID(uIdtVectorInfo))
5724	{
5725	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(uIdtVectorInfo);
5726	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(uIdtVectorInfo);
5727
5728	/*
5729	* If the event was a software interrupt (generated with INT n) or a software exception
5730	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
5731	* can handle the VM-exit and continue guest execution which will re-execute the
5732	* instruction rather than re-injecting the exception, as that can cause premature
5733	* trips to ring-3 before injection and involve TRPM which currently has no way of
5734	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
5735	* the problem).
5736	*/
5737	IEMXCPTRAISE enmRaise;
5738	IEMXCPTRAISEINFO fRaiseInfo;
5739	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5740	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5741	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5742	{
5743	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5744	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5745	}
5746	else if (VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo))
5747	{
5748	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
5749	uint8_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
5750	Assert(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT);
5751
5752	uint32_t const fIdtVectorFlags = vmxHCGetIemXcptFlags(uIdtVector, uIdtVectorType);
5753	uint32_t const fExitVectorFlags = vmxHCGetIemXcptFlags(uExitVector, uExitVectorType);
5754
5755	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5756
5757	/* Determine a vectoring #PF condition, see comment in vmxHCExitXcptPF(). */
5758	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
5759	{
5760	pVmxTransient->fVectoringPF = true;
5761	enmRaise = IEMXCPTRAISE_PREV_EVENT;
5762	}
5763	}
5764	else
5765	{
5766	/*
5767	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5768	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5769	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
5770	*/
5771	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5772	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5773	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5774	enmRaise = IEMXCPTRAISE_PREV_EVENT;
5775	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5776	}
5777
5778	/*
5779	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5780	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5781	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5782	* subsequent VM-entry would fail, see @bugref{7445}.
5783	*
5784	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception".
5785	*/
5786	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5787	&& enmRaise == IEMXCPTRAISE_PREV_EVENT
5788	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
5789	&& CPUMIsGuestNmiBlocking(pVCpu))
5790	{
5791	CPUMSetGuestNmiBlocking(pVCpu, false);
5792	}
5793
5794	switch (enmRaise)
5795	{
5796	case IEMXCPTRAISE_CURRENT_XCPT:
5797	{
5798	Log4Func(("IDT: Pending secondary Xcpt: idtinfo=%#RX64 exitinfo=%#RX64\n", uIdtVectorInfo, uExitIntInfo));
5799	Assert(rcStrict == VINF_SUCCESS);
5800	break;
5801	}
5802
5803	case IEMXCPTRAISE_PREV_EVENT:
5804	{
5805	uint32_t u32ErrCode;
5806	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(uIdtVectorInfo))
5807	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5808	else
5809	u32ErrCode = 0;
5810
5811	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see vmxHCExitXcptPF(). */
5812	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectReflect);
5813	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(uIdtVectorInfo), 0 /* cbInstr */,
5814	u32ErrCode, pVCpu->cpum.GstCtx.cr2);
5815
5816	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo,
5817	VCPU_2_VMXSTATE(pVCpu).Event.u32ErrCode));
5818	Assert(rcStrict == VINF_SUCCESS);
5819	break;
5820	}
5821
5822	case IEMXCPTRAISE_REEXEC_INSTR:
5823	Assert(rcStrict == VINF_SUCCESS);
5824	break;
5825
5826	case IEMXCPTRAISE_DOUBLE_FAULT:
5827	{
5828	/*
5829	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
5830	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
5831	*/
5832	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
5833	{
5834	pVmxTransient->fVectoringDoublePF = true;
5835	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo,
5836	pVCpu->cpum.GstCtx.cr2));
5837	rcStrict = VINF_SUCCESS;
5838	}
5839	else
5840	{
5841	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectConvertDF);
5842	vmxHCSetPendingXcptDF(pVCpu);
5843	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo,
5844	uIdtVector, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
5845	rcStrict = VINF_HM_DOUBLE_FAULT;
5846	}
5847	break;
5848	}
5849
5850	case IEMXCPTRAISE_TRIPLE_FAULT:
5851	{
5852	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector,
5853	VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
5854	rcStrict = VINF_EM_RESET;
5855	break;
5856	}
5857
5858	case IEMXCPTRAISE_CPU_HANG:
5859	{
5860	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
5861	rcStrict = VERR_EM_GUEST_CPU_HANG;
5862	break;
5863	}
5864
5865	default:
5866	{
5867	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
5868	rcStrict = VERR_VMX_IPE_2;
5869	break;
5870	}
5871	}
5872	}
5873	else if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
5874	&& !CPUMIsGuestNmiBlocking(pVCpu))
5875	{
5876	if ( VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo)
5877	&& VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo) != X86_XCPT_DF
5878	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(uExitIntInfo))
5879	{
5880	/*
5881	* Execution of IRET caused a fault when NMI blocking was in effect (i.e we're in
5882	* the guest or nested-guest NMI handler). We need to set the block-by-NMI field so
5883	* that virtual NMIs remain blocked until the IRET execution is completed.
5884	*
5885	* See Intel spec. 31.7.1.2 "Resuming Guest Software After Handling An Exception".
5886	*/
5887	CPUMSetGuestNmiBlocking(pVCpu, true);
5888	Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
5889	}
5890	else if ( pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
5891	\|\| pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
5892	\|\| pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
5893	{
5894	/*
5895	* Execution of IRET caused an EPT violation, page-modification log-full event or
5896	* SPP-related event VM-exit when NMI blocking was in effect (i.e. we're in the
5897	* guest or nested-guest NMI handler). We need to set the block-by-NMI field so
5898	* that virtual NMIs remain blocked until the IRET execution is completed.
5899	*
5900	* See Intel spec. 27.2.3 "Information about NMI unblocking due to IRET"
5901	*/
5902	if (VMX_EXIT_QUAL_EPT_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitQual))
5903	{
5904	CPUMSetGuestNmiBlocking(pVCpu, true);
5905	Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
5906	}
5907	}
5908	}
5909
5910	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
5911	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
5912	return rcStrict;
5913	}
5914
5915
5916	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5917	/**
5918	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
5919	* guest attempting to execute a VMX instruction.
5920	*
5921	* @returns Strict VBox status code (i.e. informational status codes too).
5922	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5923	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
5924	*
5925	* @param pVCpu The cross context virtual CPU structure.
5926	* @param uExitReason The VM-exit reason.
5927	*
5928	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
5929	* @remarks No-long-jump zone!!!
5930	*/
5931	static VBOXSTRICTRC vmxHCCheckExitDueToVmxInstr(PVMCPUCC pVCpu, uint32_t uExitReason)
5932	{
5933	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
5934	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
5935
5936	/*
5937	* The physical CPU would have already checked the CPU mode/code segment.
5938	* We shall just assert here for paranoia.
5939	* See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
5940	*/
5941	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
5942	Assert( !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5943	\|\| CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));
5944
5945	if (uExitReason == VMX_EXIT_VMXON)
5946	{
5947	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
5948
5949	/*
5950	* We check CR4.VMXE because it is required to be always set while in VMX operation
5951	* by physical CPUs and our CR4 read-shadow is only consulted when executing specific
5952	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
5953	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
5954	*/
5955	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
5956	{
5957	Log4Func(("CR4.VMXE is not set -> #UD\n"));
5958	vmxHCSetPendingXcptUD(pVCpu);
5959	return VINF_HM_PENDING_XCPT;
5960	}
5961	}
5962	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
5963	{
5964	/*
5965	* The guest has not entered VMX operation but attempted to execute a VMX instruction
5966	* (other than VMXON), we need to raise a #UD.
5967	*/
5968	Log4Func(("Not in VMX root mode -> #UD\n"));
5969	vmxHCSetPendingXcptUD(pVCpu);
5970	return VINF_HM_PENDING_XCPT;
5971	}
5972
5973	/* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
5974	return VINF_SUCCESS;
5975	}
5976
5977
5978	/**
5979	* Decodes the memory operand of an instruction that caused a VM-exit.
5980	*
5981	* The Exit qualification field provides the displacement field for memory
5982	* operand instructions, if any.
5983	*
5984	* @returns Strict VBox status code (i.e. informational status codes too).
5985	* @retval VINF_SUCCESS if the operand was successfully decoded.
5986	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
5987	* operand.
5988	* @param pVCpu The cross context virtual CPU structure.
5989	* @param uExitInstrInfo The VM-exit instruction information field.
5990	* @param enmMemAccess The memory operand's access type (read or write).
5991	* @param GCPtrDisp The instruction displacement field, if any. For
5992	* RIP-relative addressing pass RIP + displacement here.
5993	* @param pGCPtrMem Where to store the effective destination memory address.
5994	*
5995	* @remarks Warning! This function ASSUMES the instruction cannot be used in real or
5996	* virtual-8086 mode hence skips those checks while verifying if the
5997	* segment is valid.
5998	*/
5999	static VBOXSTRICTRC vmxHCDecodeMemOperand(PVMCPUCC pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
6000	PRTGCPTR pGCPtrMem)
6001	{
6002	Assert(pGCPtrMem);
6003	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
6004	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
6005	\| CPUMCTX_EXTRN_CR0);
6006
6007	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
6008	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
6009	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
6010
6011	VMXEXITINSTRINFO ExitInstrInfo;
6012	ExitInstrInfo.u = uExitInstrInfo;
6013	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
6014	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
6015	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
6016	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
6017	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
6018	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
6019	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
6020	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
6021	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
6022
6023	/*
6024	* Validate instruction information.
6025	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
6026	*/
6027	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
6028	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
6029	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
6030	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
6031	AssertLogRelMsgReturn(fIsMemOperand,
6032	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
6033
6034	/*
6035	* Compute the complete effective address.
6036	*
6037	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
6038	* See AMD spec. 4.5.2 "Segment Registers".
6039	*/
6040	RTGCPTR GCPtrMem = GCPtrDisp;
6041	if (fBaseRegValid)
6042	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
6043	if (fIdxRegValid)
6044	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
6045
6046	RTGCPTR const GCPtrOff = GCPtrMem;
6047	if ( !fIsLongMode
6048	\|\| iSegReg >= X86_SREG_FS)
6049	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
6050	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
6051
6052	/*
6053	* Validate effective address.
6054	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
6055	*/
6056	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
6057	Assert(cbAccess > 0);
6058	if (fIsLongMode)
6059	{
6060	if (X86_IS_CANONICAL(GCPtrMem))
6061	{
6062	*pGCPtrMem = GCPtrMem;
6063	return VINF_SUCCESS;
6064	}
6065
6066	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
6067	* "Data Limit Checks in 64-bit Mode". */
6068	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
6069	vmxHCSetPendingXcptGP(pVCpu, 0);
6070	return VINF_HM_PENDING_XCPT;
6071	}
6072
6073	/*
6074	* This is a watered down version of iemMemApplySegment().
6075	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
6076	* and segment CPL/DPL checks are skipped.
6077	*/
6078	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
6079	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
6080	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
6081
6082	/* Check if the segment is present and usable. */
6083	if ( pSel->Attr.n.u1Present
6084	&& !pSel->Attr.n.u1Unusable)
6085	{
6086	Assert(pSel->Attr.n.u1DescType);
6087	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
6088	{
6089	/* Check permissions for the data segment. */
6090	if ( enmMemAccess == VMXMEMACCESS_WRITE
6091	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
6092	{
6093	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6094	vmxHCSetPendingXcptGP(pVCpu, iSegReg);
6095	return VINF_HM_PENDING_XCPT;
6096	}
6097
6098	/* Check limits if it's a normal data segment. */
6099	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
6100	{
6101	if ( GCPtrFirst32 > pSel->u32Limit
6102	\|\| GCPtrLast32 > pSel->u32Limit)
6103	{
6104	Log4Func(("Data segment limit exceeded. "
6105	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6106	GCPtrLast32, pSel->u32Limit));
6107	if (iSegReg == X86_SREG_SS)
6108	vmxHCSetPendingXcptSS(pVCpu, 0);
6109	else
6110	vmxHCSetPendingXcptGP(pVCpu, 0);
6111	return VINF_HM_PENDING_XCPT;
6112	}
6113	}
6114	else
6115	{
6116	/* Check limits if it's an expand-down data segment.
6117	Note! The upper boundary is defined by the B bit, not the G bit! */
6118	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
6119	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
6120	{
6121	Log4Func(("Expand-down data segment limit exceeded. "
6122	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6123	GCPtrLast32, pSel->u32Limit));
6124	if (iSegReg == X86_SREG_SS)
6125	vmxHCSetPendingXcptSS(pVCpu, 0);
6126	else
6127	vmxHCSetPendingXcptGP(pVCpu, 0);
6128	return VINF_HM_PENDING_XCPT;
6129	}
6130	}
6131	}
6132	else
6133	{
6134	/* Check permissions for the code segment. */
6135	if ( enmMemAccess == VMXMEMACCESS_WRITE
6136	\|\| ( enmMemAccess == VMXMEMACCESS_READ
6137	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
6138	{
6139	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
6140	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
6141	vmxHCSetPendingXcptGP(pVCpu, 0);
6142	return VINF_HM_PENDING_XCPT;
6143	}
6144
6145	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
6146	if ( GCPtrFirst32 > pSel->u32Limit
6147	\|\| GCPtrLast32 > pSel->u32Limit)
6148	{
6149	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
6150	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
6151	if (iSegReg == X86_SREG_SS)
6152	vmxHCSetPendingXcptSS(pVCpu, 0);
6153	else
6154	vmxHCSetPendingXcptGP(pVCpu, 0);
6155	return VINF_HM_PENDING_XCPT;
6156	}
6157	}
6158	}
6159	else
6160	{
6161	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6162	vmxHCSetPendingXcptGP(pVCpu, 0);
6163	return VINF_HM_PENDING_XCPT;
6164	}
6165
6166	*pGCPtrMem = GCPtrMem;
6167	return VINF_SUCCESS;
6168	}
6169	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
6170
6171
6172	/**
6173	* VM-exit helper for LMSW.
6174	*/
6175	static VBOXSTRICTRC vmxHCExitLmsw(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst)
6176	{
6177	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
6178	AssertRCReturn(rc, rc);
6179
6180	VBOXSTRICTRC rcStrict = IEMExecDecodedLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst);
6181	AssertMsg( rcStrict == VINF_SUCCESS
6182	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
6183
6184	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
6185	if (rcStrict == VINF_IEM_RAISED_XCPT)
6186	{
6187	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6188	rcStrict = VINF_SUCCESS;
6189	}
6190
6191	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitLmsw);
6192	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
6193	return rcStrict;
6194	}
6195
6196
6197	/**
6198	* VM-exit helper for CLTS.
6199	*/
6200	static VBOXSTRICTRC vmxHCExitClts(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr)
6201	{
6202	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
6203	AssertRCReturn(rc, rc);
6204
6205	VBOXSTRICTRC rcStrict = IEMExecDecodedClts(pVCpu, cbInstr);
6206	AssertMsg( rcStrict == VINF_SUCCESS
6207	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
6208
6209	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
6210	if (rcStrict == VINF_IEM_RAISED_XCPT)
6211	{
6212	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6213	rcStrict = VINF_SUCCESS;
6214	}
6215
6216	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitClts);
6217	Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
6218	return rcStrict;
6219	}
6220
6221
6222	/**
6223	* VM-exit helper for MOV from CRx (CRx read).
6224	*/
6225	static VBOXSTRICTRC vmxHCExitMovFromCrX(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
6226	{
6227	Assert(iCrReg < 16);
6228	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
6229
6230	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
6231	AssertRCReturn(rc, rc);
6232
6233	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
6234	AssertMsg( rcStrict == VINF_SUCCESS
6235	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
6236
6237	if (iGReg == X86_GREG_xSP)
6238	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
6239	else
6240	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
6241	#ifdef VBOX_WITH_STATISTICS
6242	switch (iCrReg)
6243	{
6244	case 0: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR0Read); break;
6245	case 2: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR2Read); break;
6246	case 3: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR3Read); break;
6247	case 4: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR4Read); break;
6248	case 8: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR8Read); break;
6249	}
6250	#endif
6251	Log4Func(("CR%d Read access rcStrict=%Rrc\n", iCrReg, VBOXSTRICTRC_VAL(rcStrict)));
6252	return rcStrict;
6253	}
6254
6255
6256	/**
6257	* VM-exit helper for MOV to CRx (CRx write).
6258	*/
6259	static VBOXSTRICTRC vmxHCExitMovToCrX(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
6260	{
6261	HMVMX_CPUMCTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6262
6263	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
6264	AssertMsg( rcStrict == VINF_SUCCESS
6265	\|\| rcStrict == VINF_IEM_RAISED_XCPT
6266	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
6267
6268	switch (iCrReg)
6269	{
6270	case 0:
6271	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0
6272	\| HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
6273	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR0Write);
6274	Log4Func(("CR0 write. rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
6275	break;
6276
6277	case 2:
6278	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR2Write);
6279	/* Nothing to do here, CR2 it's not part of the VMCS. */
6280	break;
6281
6282	case 3:
6283	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
6284	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR3Write);
6285	Log4Func(("CR3 write. rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
6286	break;
6287
6288	case 4:
6289	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
6290	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR4Write);
6291	#ifndef IN_NEM_DARWIN
6292	Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
6293	pVCpu->cpum.GstCtx.cr4, pVCpu->hmr0.s.fLoadSaveGuestXcr0));
6294	#else
6295	Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr4));
6296	#endif
6297	break;
6298
6299	case 8:
6300	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged,
6301	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
6302	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR8Write);
6303	break;
6304
6305	default:
6306	AssertMsgFailed(("Invalid CRx register %#x\n", iCrReg));
6307	break;
6308	}
6309
6310	if (rcStrict == VINF_IEM_RAISED_XCPT)
6311	{
6312	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6313	rcStrict = VINF_SUCCESS;
6314	}
6315	return rcStrict;
6316	}
6317
6318
6319	/**
6320	* VM-exit exception handler for \#PF (Page-fault exception).
6321	*
6322	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
6323	*/
6324	static VBOXSTRICTRC vmxHCExitXcptPF(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6325	{
6326	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
6327	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
6328
6329	#ifndef IN_NEM_DARWIN
6330	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6331	if (!VM_IS_VMX_NESTED_PAGING(pVM))
6332	{ /* likely */ }
6333	else
6334	#endif
6335	{
6336	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF) && !defined(IN_NEM_DARWIN)
6337	Assert(pVmxTransient->fIsNestedGuest \|\| pVCpu->hmr0.s.fUsingDebugLoop);
6338	#endif
6339	VCPU_2_VMXSTATE(pVCpu).Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
6340	if (!pVmxTransient->fVectoringDoublePF)
6341	{
6342	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
6343	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
6344	}
6345	else
6346	{
6347	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
6348	Assert(!pVmxTransient->fIsNestedGuest);
6349	vmxHCSetPendingXcptDF(pVCpu);
6350	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
6351	}
6352	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestPF);
6353	return VINF_SUCCESS;
6354	}
6355
6356	Assert(!pVmxTransient->fIsNestedGuest);
6357
6358	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
6359	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
6360	if (pVmxTransient->fVectoringPF)
6361	{
6362	Assert(VCPU_2_VMXSTATE(pVCpu).Event.fPending);
6363	return VINF_EM_RAW_INJECT_TRPM_EVENT;
6364	}
6365
6366	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6367	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
6368	AssertRCReturn(rc, rc);
6369
6370	Log4Func(("#PF: cs:rip=%#04x:%#RX64 err_code=%#RX32 exit_qual=%#RX64 cr3=%#RX64\n", pCtx->cs.Sel, pCtx->rip,
6371	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual, pCtx->cr3));
6372
6373	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
6374	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
6375
6376	Log4Func(("#PF: rc=%Rrc\n", rc));
6377	if (rc == VINF_SUCCESS)
6378	{
6379	/*
6380	* This is typically a shadow page table sync or a MMIO instruction. But we may have
6381	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
6382	*/
6383	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
6384	TRPMResetTrap(pVCpu);
6385	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitShadowPF);
6386	return rc;
6387	}
6388
6389	if (rc == VINF_EM_RAW_GUEST_TRAP)
6390	{
6391	if (!pVmxTransient->fVectoringDoublePF)
6392	{
6393	/* It's a guest page fault and needs to be reflected to the guest. */
6394	uint32_t const uGstErrorCode = TRPMGetErrorCode(pVCpu);
6395	TRPMResetTrap(pVCpu);
6396	VCPU_2_VMXSTATE(pVCpu).Event.fPending = false; /* In case it's a contributory #PF. */
6397	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
6398	uGstErrorCode, pVmxTransient->uExitQual);
6399	}
6400	else
6401	{
6402	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
6403	TRPMResetTrap(pVCpu);
6404	VCPU_2_VMXSTATE(pVCpu).Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
6405	vmxHCSetPendingXcptDF(pVCpu);
6406	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
6407	}
6408
6409	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestPF);
6410	return VINF_SUCCESS;
6411	}
6412
6413	TRPMResetTrap(pVCpu);
6414	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitShadowPFEM);
6415	return rc;
6416	}
6417
6418
6419	/**
6420	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
6421	*
6422	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
6423	*/
6424	static VBOXSTRICTRC vmxHCExitXcptMF(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6425	{
6426	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
6427	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestMF);
6428
6429	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
6430	AssertRCReturn(rc, rc);
6431
6432	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
6433	{
6434	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
6435	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
6436
6437	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
6438	* provides VM-exit instruction length. If this causes problem later,
6439	* disassemble the instruction like it's done on AMD-V. */
6440	int rc2 = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
6441	AssertRCReturn(rc2, rc2);
6442	return rc;
6443	}
6444
6445	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbExitInstr,
6446	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
6447	return VINF_SUCCESS;
6448	}
6449
6450
6451	/**
6452	* VM-exit exception handler for \#BP (Breakpoint exception).
6453	*
6454	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
6455	*/
6456	static VBOXSTRICTRC vmxHCExitXcptBP(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6457	{
6458	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
6459	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestBP);
6460
6461	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
6462	AssertRCReturn(rc, rc);
6463
6464	VBOXSTRICTRC rcStrict;
6465	if (!pVmxTransient->fIsNestedGuest)
6466	rcStrict = DBGFTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx));
6467	else
6468	rcStrict = VINF_EM_RAW_GUEST_TRAP;
6469
6470	if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
6471	{
6472	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
6473	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
6474	rcStrict = VINF_SUCCESS;
6475	}
6476
6477	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_DBG_BREAKPOINT);
6478	return rcStrict;
6479	}
6480
6481
6482	/**
6483	* VM-exit exception handler for \#AC (Alignment-check exception).
6484	*
6485	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
6486	*/
6487	static VBOXSTRICTRC vmxHCExitXcptAC(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6488	{
6489	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
6490
6491	/*
6492	* Detect #ACs caused by host having enabled split-lock detection.
6493	* Emulate such instructions.
6494	*/
6495	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo,
6496	CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS);
6497	AssertRCReturn(rc, rc);
6498	/** @todo detect split lock in cpu feature? */
6499	if ( /* 1. If 486-style alignment checks aren't enabled, then this must be a split-lock exception */
6500	!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_AM)
6501	/* 2. #AC cannot happen in rings 0-2 except for split-lock detection. */
6502	\|\| CPUMGetGuestCPL(pVCpu) != 3
6503	/* 3. When the EFLAGS.AC != 0 this can only be a split-lock case. */
6504	\|\| !(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_AC) )
6505	{
6506	/*
6507	* Check for debug/trace events and import state accordingly.
6508	*/
6509	STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestACSplitLock);
6510	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6511	if ( !DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_VMX_SPLIT_LOCK)
6512	#ifndef IN_NEM_DARWIN
6513	&& !VBOXVMM_VMX_SPLIT_LOCK_ENABLED()
6514	#endif
6515	)
6516	{
6517	if (pVM->cCpus == 1)
6518	{
6519	#if 0 /** @todo r=bird: This is potentially wrong. Might have to just do a whole state sync above and mark everything changed to be safe... */
6520	rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
6521	#else
6522	rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
6523	#endif
6524	AssertRCReturn(rc, rc);
6525	}
6526	}
6527	else
6528	{
6529	rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
6530	AssertRCReturn(rc, rc);
6531
6532	VBOXVMM_XCPT_DF(pVCpu, &pVCpu->cpum.GstCtx);
6533
6534	if (DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_VMX_SPLIT_LOCK))
6535	{
6536	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, DBGFEVENT_VMX_SPLIT_LOCK, DBGFEVENTCTX_HM, 0);
6537	if (rcStrict != VINF_SUCCESS)
6538	return rcStrict;
6539	}
6540	}
6541
6542	/*
6543	* Emulate the instruction.
6544	*
6545	* We have to ignore the LOCK prefix here as we must not retrigger the
6546	* detection on the host. This isn't all that satisfactory, though...
6547	*/
6548	if (pVM->cCpus == 1)
6549	{
6550	Log8Func(("cs:rip=%#04x:%#RX64 rflags=%#RX64 cr0=%#RX64 split-lock #AC\n", pVCpu->cpum.GstCtx.cs.Sel,
6551	pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags, pVCpu->cpum.GstCtx.cr0));
6552
6553	/** @todo For SMP configs we should do a rendezvous here. */
6554	VBOXSTRICTRC rcStrict = IEMExecOneIgnoreLock(pVCpu);
6555	if (rcStrict == VINF_SUCCESS)
6556	#if 0 /** @todo r=bird: This is potentially wrong. Might have to just do a whole state sync above and mark everything changed to be safe... */
6557	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged,
6558	HM_CHANGED_GUEST_RIP
6559	\| HM_CHANGED_GUEST_RFLAGS
6560	\| HM_CHANGED_GUEST_GPRS_MASK
6561	\| HM_CHANGED_GUEST_CS
6562	\| HM_CHANGED_GUEST_SS);
6563	#else
6564	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
6565	#endif
6566	else if (rcStrict == VINF_IEM_RAISED_XCPT)
6567	{
6568	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6569	rcStrict = VINF_SUCCESS;
6570	}
6571	return rcStrict;
6572	}
6573	Log8Func(("cs:rip=%#04x:%#RX64 rflags=%#RX64 cr0=%#RX64 split-lock #AC -> VINF_EM_EMULATE_SPLIT_LOCK\n",
6574	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags, pVCpu->cpum.GstCtx.cr0));
6575	return VINF_EM_EMULATE_SPLIT_LOCK;
6576	}
6577
6578	STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestAC);
6579	Log8Func(("cs:rip=%#04x:%#RX64 rflags=%#RX64 cr0=%#RX64 cpl=%d -> #AC\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
6580	pVCpu->cpum.GstCtx.rflags, pVCpu->cpum.GstCtx.cr0, CPUMGetGuestCPL(pVCpu) ));
6581
6582	/* Re-inject it. We'll detect any nesting before getting here. */
6583	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
6584	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
6585	return VINF_SUCCESS;
6586	}
6587
6588
6589	/**
6590	* VM-exit exception handler for \#DB (Debug exception).
6591	*
6592	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
6593	*/
6594	static VBOXSTRICTRC vmxHCExitXcptDB(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6595	{
6596	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
6597	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDB);
6598
6599	/*
6600	* Get the DR6-like values from the Exit qualification and pass it to DBGF for processing.
6601	*/
6602	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
6603
6604	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
6605	uint64_t const uDR6 = X86_DR6_INIT_VAL
6606	\| (pVmxTransient->uExitQual & ( X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3
6607	\| X86_DR6_BD \| X86_DR6_BS));
6608
6609	int rc;
6610	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6611	if (!pVmxTransient->fIsNestedGuest)
6612	{
6613	rc = DBGFTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, VCPU_2_VMXSTATE(pVCpu).fSingleInstruction);
6614
6615	/*
6616	* Prevents stepping twice over the same instruction when the guest is stepping using
6617	* EFLAGS.TF and the hypervisor debugger is stepping using MTF.
6618	* Testcase: DOSQEMM, break (using "ba x 1") at cs:rip 0x70:0x774 and step (using "t").
6619	*/
6620	if ( rc == VINF_EM_DBG_STEPPED
6621	&& (pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MONITOR_TRAP_FLAG))
6622	{
6623	Assert(VCPU_2_VMXSTATE(pVCpu).fSingleInstruction);
6624	rc = VINF_EM_RAW_GUEST_TRAP;
6625	}
6626	}
6627	else
6628	rc = VINF_EM_RAW_GUEST_TRAP;
6629	Log6Func(("rc=%Rrc\n", rc));
6630	if (rc == VINF_EM_RAW_GUEST_TRAP)
6631	{
6632	/*
6633	* The exception was for the guest. Update DR6, DR7.GD and
6634	* IA32_DEBUGCTL.LBR before forwarding it.
6635	* See Intel spec. 27.1 "Architectural State before a VM-Exit".
6636	*/
6637	#ifndef IN_NEM_DARWIN
6638	VMMRZCallRing3Disable(pVCpu);
6639	HM_DISABLE_PREEMPT(pVCpu);
6640
6641	pCtx->dr[6] &= ~X86_DR6_B_MASK;
6642	pCtx->dr[6] \|= uDR6;
6643	if (CPUMIsGuestDebugStateActive(pVCpu))
6644	ASMSetDR6(pCtx->dr[6]);
6645
6646	HM_RESTORE_PREEMPT();
6647	VMMRZCallRing3Enable(pVCpu);
6648	#else
6649	/** @todo */
6650	#endif
6651
6652	rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
6653	AssertRCReturn(rc, rc);
6654
6655	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
6656	pCtx->dr[7] &= ~(uint64_t)X86_DR7_GD;
6657
6658	/* Paranoia. */
6659	pCtx->dr[7] &= ~(uint64_t)X86_DR7_RAZ_MASK;
6660	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
6661
6662	rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
6663	AssertRC(rc);
6664
6665	/*
6666	* Raise #DB in the guest.
6667	*
6668	* It is important to reflect exactly what the VM-exit gave us (preserving the
6669	* interruption-type) rather than use vmxHCSetPendingXcptDB() as the #DB could've
6670	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
6671	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
6672	*
6673	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
6674	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6675	*/
6676	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
6677	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
6678	return VINF_SUCCESS;
6679	}
6680
6681	/*
6682	* Not a guest trap, must be a hypervisor related debug event then.
6683	* Update DR6 in case someone is interested in it.
6684	*/
6685	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
6686	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
6687	CPUMSetHyperDR6(pVCpu, uDR6);
6688
6689	return rc;
6690	}
6691
6692
6693	/**
6694	* Hacks its way around the lovely mesa driver's backdoor accesses.
6695	*
6696	* @sa hmR0SvmHandleMesaDrvGp.
6697	*/
6698	static int vmxHCHandleMesaDrvGp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
6699	{
6700	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
6701	RT_NOREF(pCtx);
6702
6703	/* For now we'll just skip the instruction. */
6704	return vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
6705	}
6706
6707
6708	/**
6709	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
6710	* backdoor logging w/o checking what it is running inside.
6711	*
6712	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
6713	* backdoor port and magic numbers loaded in registers.
6714	*
6715	* @returns true if it is, false if it isn't.
6716	* @sa hmR0SvmIsMesaDrvGp.
6717	*/
6718	DECLINLINE(bool) vmxHCIsMesaDrvGp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
6719	{
6720	/* 0xed: IN eAX,dx */
6721	uint8_t abInstr[1];
6722	if (pVmxTransient->cbExitInstr != sizeof(abInstr))
6723	return false;
6724
6725	/* Check that it is #GP(0). */
6726	if (pVmxTransient->uExitIntErrorCode != 0)
6727	return false;
6728
6729	/* Check magic and port. */
6730	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
6731	/Log(("vmxHCIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
6732	if (pCtx->rax != UINT32_C(0x564d5868))
6733	return false;
6734	if (pCtx->dx != UINT32_C(0x5658))
6735	return false;
6736
6737	/* Flat ring-3 CS. */
6738	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
6739	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
6740	/Log(("vmxHCIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
6741	if (pCtx->cs.Attr.n.u2Dpl != 3)
6742	return false;
6743	if (pCtx->cs.u64Base != 0)
6744	return false;
6745
6746	/* Check opcode. */
6747	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
6748	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
6749	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
6750	/Log(("vmxHCIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
6751	if (RT_FAILURE(rc))
6752	return false;
6753	if (abInstr[0] != 0xed)
6754	return false;
6755
6756	return true;
6757	}
6758
6759
6760	/**
6761	* VM-exit exception handler for \#GP (General-protection exception).
6762	*
6763	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
6764	*/
6765	static VBOXSTRICTRC vmxHCExitXcptGP(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6766	{
6767	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
6768	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestGP);
6769
6770	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6771	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6772	#ifndef IN_NEM_DARWIN
6773	PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
6774	if (pVmcsInfoShared->RealMode.fRealOnV86Active)
6775	{ /* likely */ }
6776	else
6777	#endif
6778	{
6779	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
6780	# ifndef IN_NEM_DARWIN
6781	Assert(pVCpu->hmr0.s.fUsingDebugLoop \|\| VCPU_2_VMXSTATE(pVCpu).fTrapXcptGpForLovelyMesaDrv \|\| pVmxTransient->fIsNestedGuest);
6782	# else
6783	Assert(/pVCpu->hmr0.s.fUsingDebugLoop \|\|/ VCPU_2_VMXSTATE(pVCpu).fTrapXcptGpForLovelyMesaDrv \|\| pVmxTransient->fIsNestedGuest);
6784	# endif
6785	#endif
6786	/*
6787	* If the guest is not in real-mode or we have unrestricted guest execution support, or if we are
6788	* executing a nested-guest, reflect #GP to the guest or nested-guest.
6789	*/
6790	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
6791	AssertRCReturn(rc, rc);
6792	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
6793	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
6794
6795	if ( pVmxTransient->fIsNestedGuest
6796	\|\| !VCPU_2_VMXSTATE(pVCpu).fTrapXcptGpForLovelyMesaDrv
6797	\|\| !vmxHCIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
6798	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
6799	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
6800	else
6801	rc = vmxHCHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
6802	return rc;
6803	}
6804
6805	#ifndef IN_NEM_DARWIN
6806	Assert(CPUMIsGuestInRealModeEx(pCtx));
6807	Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fUnrestrictedGuest);
6808	Assert(!pVmxTransient->fIsNestedGuest);
6809
6810	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
6811	AssertRCReturn(rc, rc);
6812
6813	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
6814	if (rcStrict == VINF_SUCCESS)
6815	{
6816	if (!CPUMIsGuestInRealModeEx(pCtx))
6817	{
6818	/*
6819	* The guest is no longer in real-mode, check if we can continue executing the
6820	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
6821	*/
6822	pVmcsInfoShared->RealMode.fRealOnV86Active = false;
6823	if (HMCanExecuteVmxGuest(pVCpu->CTX_SUFF(pVM), pVCpu, pCtx))
6824	{
6825	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
6826	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
6827	}
6828	else
6829	{
6830	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
6831	rcStrict = VINF_EM_RESCHEDULE;
6832	}
6833	}
6834	else
6835	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
6836	}
6837	else if (rcStrict == VINF_IEM_RAISED_XCPT)
6838	{
6839	rcStrict = VINF_SUCCESS;
6840	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6841	}
6842	return VBOXSTRICTRC_VAL(rcStrict);
6843	#endif
6844	}
6845
6846
6847	/**
6848	* VM-exit exception handler wrapper for all other exceptions that are not handled
6849	* by a specific handler.
6850	*
6851	* This simply re-injects the exception back into the VM without any special
6852	* processing.
6853	*
6854	* @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
6855	*/
6856	static VBOXSTRICTRC vmxHCExitXcptOthers(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6857	{
6858	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
6859
6860	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
6861	# ifndef IN_NEM_DARWIN
6862	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6863	AssertMsg(pVCpu->hmr0.s.fUsingDebugLoop \|\| pVmcsInfo->pShared->RealMode.fRealOnV86Active \|\| pVmxTransient->fIsNestedGuest,
6864	("uVector=%#x u32XcptBitmap=%#X32\n",
6865	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
6866	NOREF(pVmcsInfo);
6867	# endif
6868	#endif
6869
6870	/*
6871	* Re-inject the exception into the guest. This cannot be a double-fault condition which
6872	* would have been handled while checking exits due to event delivery.
6873	*/
6874	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
6875
6876	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
6877	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6878	AssertRCReturn(rc, rc);
6879	Log4Func(("Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
6880	#endif
6881
6882	#ifdef VBOX_WITH_STATISTICS
6883	switch (uVector)
6884	{
6885	case X86_XCPT_DE: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDE); break;
6886	case X86_XCPT_DB: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDB); break;
6887	case X86_XCPT_BP: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestBP); break;
6888	case X86_XCPT_OF: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestOF); break;
6889	case X86_XCPT_BR: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestBR); break;
6890	case X86_XCPT_UD: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestUD); break;
6891	case X86_XCPT_NM: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestOF); break;
6892	case X86_XCPT_DF: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDF); break;
6893	case X86_XCPT_TS: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestTS); break;
6894	case X86_XCPT_NP: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestNP); break;
6895	case X86_XCPT_SS: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestSS); break;
6896	case X86_XCPT_GP: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestGP); break;
6897	case X86_XCPT_PF: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestPF); break;
6898	case X86_XCPT_MF: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestMF); break;
6899	case X86_XCPT_AC: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestAC); break;
6900	case X86_XCPT_XF: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestXF); break;
6901	default:
6902	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestXcpUnk);
6903	break;
6904	}
6905	#endif
6906
6907	/* We should never call this function for a page-fault, we'd need to pass on the fault address below otherwise. */
6908	Assert(!VMX_EXIT_INT_INFO_IS_XCPT_PF(pVmxTransient->uExitIntInfo));
6909	NOREF(uVector);
6910
6911	/* Re-inject the original exception into the guest. */
6912	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
6913	pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
6914	return VINF_SUCCESS;
6915	}
6916
6917
6918	/**
6919	* VM-exit exception handler for all exceptions (except NMIs!).
6920	*
6921	* @remarks This may be called for both guests and nested-guests. Take care to not
6922	* make assumptions and avoid doing anything that is not relevant when
6923	* executing a nested-guest (e.g., Mesa driver hacks).
6924	*/
6925	static VBOXSTRICTRC vmxHCExitXcpt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6926	{
6927	HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
6928
6929	/*
6930	* If this VM-exit occurred while delivering an event through the guest IDT, take
6931	* action based on the return code and additional hints (e.g. for page-faults)
6932	* that will be updated in the VMX transient structure.
6933	*/
6934	VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
6935	if (rcStrict == VINF_SUCCESS)
6936	{
6937	/*
6938	* If an exception caused a VM-exit due to delivery of an event, the original
6939	* event may have to be re-injected into the guest. We shall reinject it and
6940	* continue guest execution. However, page-fault is a complicated case and
6941	* needs additional processing done in vmxHCExitXcptPF().
6942	*/
6943	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
6944	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
6945	if ( !VCPU_2_VMXSTATE(pVCpu).Event.fPending
6946	\|\| uVector == X86_XCPT_PF)
6947	{
6948	switch (uVector)
6949	{
6950	case X86_XCPT_PF: return vmxHCExitXcptPF(pVCpu, pVmxTransient);
6951	case X86_XCPT_GP: return vmxHCExitXcptGP(pVCpu, pVmxTransient);
6952	case X86_XCPT_MF: return vmxHCExitXcptMF(pVCpu, pVmxTransient);
6953	case X86_XCPT_DB: return vmxHCExitXcptDB(pVCpu, pVmxTransient);
6954	case X86_XCPT_BP: return vmxHCExitXcptBP(pVCpu, pVmxTransient);
6955	case X86_XCPT_AC: return vmxHCExitXcptAC(pVCpu, pVmxTransient);
6956	default:
6957	return vmxHCExitXcptOthers(pVCpu, pVmxTransient);
6958	}
6959	}
6960	/* else: inject pending event before resuming guest execution. */
6961	}
6962	else if (rcStrict == VINF_HM_DOUBLE_FAULT)
6963	{
6964	Assert(VCPU_2_VMXSTATE(pVCpu).Event.fPending);
6965	rcStrict = VINF_SUCCESS;
6966	}
6967
6968	return rcStrict;
6969	}
6970	/** @} */
6971
6972
6973	/** @name VM-exit handlers.
6974	* @{
6975	*/
6976	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6977	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6978	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6979
6980	/**
6981	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
6982	*/
6983	HMVMX_EXIT_DECL vmxHCExitExtInt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
6984	{
6985	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
6986	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitExtInt);
6987
6988	#ifndef IN_NEM_DARWIN
6989	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
6990	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
6991	return VINF_SUCCESS;
6992	return VINF_EM_RAW_INTERRUPT;
6993	#else
6994	return VINF_SUCCESS;
6995	#endif
6996	}
6997
6998
6999	/**
7000	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI). Conditional
7001	* VM-exit.
7002	*/
7003	HMVMX_EXIT_DECL vmxHCExitXcptOrNmi(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7004	{
7005	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7006	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitXcptNmi, y3);
7007
7008	vmxHCReadExitIntInfoVmcs(pVCpu, pVmxTransient);
7009
7010	uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
7011	uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
7012	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
7013
7014	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7015	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
7016	&& uExitIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
7017	NOREF(pVmcsInfo);
7018
7019	VBOXSTRICTRC rcStrict;
7020	switch (uExitIntType)
7021	{
7022	#ifndef IN_NEM_DARWIN /* NMIs should never reach R3. */
7023	/*
7024	* Host physical NMIs:
7025	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
7026	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
7027	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
7028	*
7029	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
7030	* See Intel spec. 27.5.5 "Updating Non-Register State".
7031	*/
7032	case VMX_EXIT_INT_INFO_TYPE_NMI:
7033	{
7034	rcStrict = hmR0VmxExitHostNmi(pVCpu, pVmcsInfo);
7035	break;
7036	}
7037	#endif
7038
7039	/*
7040	* Privileged software exceptions (#DB from ICEBP),
7041	* Software exceptions (#BP and #OF),
7042	* Hardware exceptions:
7043	* Process the required exceptions and resume guest execution if possible.
7044	*/
7045	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
7046	Assert(uVector == X86_XCPT_DB);
7047	RT_FALL_THRU();
7048	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
7049	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uExitIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
7050	RT_FALL_THRU();
7051	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
7052	{
7053	NOREF(uVector);
7054	vmxHCReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
7055	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7056	vmxHCReadIdtVectoringInfoVmcs(pVCpu, pVmxTransient);
7057	vmxHCReadIdtVectoringErrorCodeVmcs(pVCpu, pVmxTransient);
7058
7059	rcStrict = vmxHCExitXcpt(pVCpu, pVmxTransient);
7060	break;
7061	}
7062
7063	default:
7064	{
7065	VCPU_2_VMXSTATE(pVCpu).u32HMError = pVmxTransient->uExitIntInfo;
7066	rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
7067	AssertMsgFailed(("Invalid/unexpected VM-exit interruption info %#x\n", pVmxTransient->uExitIntInfo));
7068	break;
7069	}
7070	}
7071
7072	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitXcptNmi, y3);
7073	return rcStrict;
7074	}
7075
7076
7077	/**
7078	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
7079	*/
7080	HMVMX_EXIT_NSRC_DECL vmxHCExitIntWindow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7081	{
7082	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7083
7084	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
7085	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7086	vmxHCClearIntWindowExitVmcs(pVCpu, pVmcsInfo);
7087
7088	/* Evaluate and deliver pending events and resume guest execution. */
7089	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitIntWindow);
7090	return VINF_SUCCESS;
7091	}
7092
7093
7094	/**
7095	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
7096	*/
7097	HMVMX_EXIT_NSRC_DECL vmxHCExitNmiWindow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7098	{
7099	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7100
7101	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7102	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
7103	{
7104	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
7105	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
7106	}
7107
7108	Assert(!CPUMIsGuestNmiBlocking(pVCpu));
7109
7110	/*
7111	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
7112	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
7113	*/
7114	uint32_t fIntrState;
7115	int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
7116	AssertRC(rc);
7117	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
7118	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
7119	{
7120	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7121	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7122
7123	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
7124	rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, fIntrState);
7125	AssertRC(rc);
7126	}
7127
7128	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
7129	vmxHCClearNmiWindowExitVmcs(pVCpu, pVmcsInfo);
7130
7131	/* Evaluate and deliver pending events and resume guest execution. */
7132	return VINF_SUCCESS;
7133	}
7134
7135
7136	/**
7137	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
7138	*/
7139	HMVMX_EXIT_NSRC_DECL vmxHCExitWbinvd(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7140	{
7141	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7142	return vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
7143	}
7144
7145
7146	/**
7147	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
7148	*/
7149	HMVMX_EXIT_NSRC_DECL vmxHCExitInvd(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7150	{
7151	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7152	return vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
7153	}
7154
7155
7156	/**
7157	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
7158	*/
7159	HMVMX_EXIT_DECL vmxHCExitCpuid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7160	{
7161	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7162
7163	/*
7164	* Get the state we need and update the exit history entry.
7165	*/
7166	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7167	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7168
7169	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
7170	AssertRCReturn(rc, rc);
7171
7172	VBOXSTRICTRC rcStrict;
7173	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
7174	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
7175	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
7176	if (!pExitRec)
7177	{
7178	/*
7179	* Regular CPUID instruction execution.
7180	*/
7181	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbExitInstr);
7182	if (rcStrict == VINF_SUCCESS)
7183	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
7184	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7185	{
7186	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7187	rcStrict = VINF_SUCCESS;
7188	}
7189	}
7190	else
7191	{
7192	/*
7193	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
7194	*/
7195	int rc2 = vmxHCImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
7196	AssertRCReturn(rc2, rc2);
7197
7198	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
7199	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
7200
7201	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
7202	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
7203
7204	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
7205	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
7206	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
7207	}
7208	return rcStrict;
7209	}
7210
7211
7212	/**
7213	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
7214	*/
7215	HMVMX_EXIT_DECL vmxHCExitGetsec(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7216	{
7217	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7218
7219	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7220	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
7221	AssertRCReturn(rc, rc);
7222
7223	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
7224	return VINF_EM_RAW_EMULATE_INSTR;
7225
7226	AssertMsgFailed(("vmxHCExitGetsec: Unexpected VM-exit when CR4.SMXE is 0.\n"));
7227	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
7228	}
7229
7230
7231	/**
7232	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
7233	*/
7234	HMVMX_EXIT_DECL vmxHCExitRdtsc(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7235	{
7236	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7237
7238	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7239	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7240	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
7241	AssertRCReturn(rc, rc);
7242
7243	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbExitInstr);
7244	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
7245	{
7246	/* If we get a spurious VM-exit when TSC offsetting is enabled,
7247	we must reset offsetting on VM-entry. See @bugref{6634}. */
7248	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
7249	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
7250	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
7251	}
7252	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7253	{
7254	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7255	rcStrict = VINF_SUCCESS;
7256	}
7257	return rcStrict;
7258	}
7259
7260
7261	/**
7262	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7263	*/
7264	HMVMX_EXIT_DECL vmxHCExitRdtscp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7265	{
7266	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7267
7268	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7269	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7270	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
7271	AssertRCReturn(rc, rc);
7272
7273	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbExitInstr);
7274	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
7275	{
7276	/* If we get a spurious VM-exit when TSC offsetting is enabled,
7277	we must reset offsetting on VM-reentry. See @bugref{6634}. */
7278	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
7279	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
7280	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
7281	}
7282	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7283	{
7284	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7285	rcStrict = VINF_SUCCESS;
7286	}
7287	return rcStrict;
7288	}
7289
7290
7291	/**
7292	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7293	*/
7294	HMVMX_EXIT_DECL vmxHCExitRdpmc(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7295	{
7296	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7297
7298	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7299	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
7300	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
7301	AssertRCReturn(rc, rc);
7302
7303	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7304	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
7305	if (RT_LIKELY(rc == VINF_SUCCESS))
7306	{
7307	rc = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
7308	Assert(pVmxTransient->cbExitInstr == 2);
7309	}
7310	else
7311	{
7312	AssertMsgFailed(("vmxHCExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7313	rc = VERR_EM_INTERPRETER;
7314	}
7315	return rc;
7316	}
7317
7318
7319	/**
7320	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
7321	*/
7322	HMVMX_EXIT_DECL vmxHCExitVmcall(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7323	{
7324	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7325
7326	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
7327	if (EMAreHypercallInstructionsEnabled(pVCpu))
7328	{
7329	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7330	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
7331	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
7332	AssertRCReturn(rc, rc);
7333
7334	/* Perform the hypercall. */
7335	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
7336	if (rcStrict == VINF_SUCCESS)
7337	{
7338	rc = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
7339	AssertRCReturn(rc, rc);
7340	}
7341	else
7342	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
7343	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
7344	\|\| RT_FAILURE(rcStrict));
7345
7346	/* If the hypercall changes anything other than guest's general-purpose registers,
7347	we would need to reload the guest changed bits here before VM-entry. */
7348	}
7349	else
7350	Log4Func(("Hypercalls not enabled\n"));
7351
7352	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
7353	if (RT_FAILURE(rcStrict))
7354	{
7355	vmxHCSetPendingXcptUD(pVCpu);
7356	rcStrict = VINF_SUCCESS;
7357	}
7358
7359	return rcStrict;
7360	}
7361
7362
7363	/**
7364	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7365	*/
7366	HMVMX_EXIT_DECL vmxHCExitInvlpg(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7367	{
7368	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7369	#ifndef IN_NEM_DARWIN
7370	Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging \|\| pVCpu->hmr0.s.fUsingDebugLoop);
7371	#endif
7372
7373	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7374	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
7375	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7376	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
7377	AssertRCReturn(rc, rc);
7378
7379	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbExitInstr, pVmxTransient->uExitQual);
7380
7381	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
7382	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
7383	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7384	{
7385	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7386	rcStrict = VINF_SUCCESS;
7387	}
7388	else
7389	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) status: %Rrc\n", pVmxTransient->uExitQual,
7390	VBOXSTRICTRC_VAL(rcStrict)));
7391	return rcStrict;
7392	}
7393
7394
7395	/**
7396	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7397	*/
7398	HMVMX_EXIT_DECL vmxHCExitMonitor(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7399	{
7400	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7401
7402	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7403	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7404	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
7405	AssertRCReturn(rc, rc);
7406
7407	VBOXSTRICTRC rcStrict = IEMExecDecodedMonitor(pVCpu, pVmxTransient->cbExitInstr);
7408	if (rcStrict == VINF_SUCCESS)
7409	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
7410	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7411	{
7412	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7413	rcStrict = VINF_SUCCESS;
7414	}
7415
7416	return rcStrict;
7417	}
7418
7419
7420	/**
7421	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7422	*/
7423	HMVMX_EXIT_DECL vmxHCExitMwait(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7424	{
7425	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7426
7427	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7428	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7429	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
7430	AssertRCReturn(rc, rc);
7431
7432	VBOXSTRICTRC rcStrict = IEMExecDecodedMwait(pVCpu, pVmxTransient->cbExitInstr);
7433	if (RT_SUCCESS(rcStrict))
7434	{
7435	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
7436	if (EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
7437	rcStrict = VINF_SUCCESS;
7438	}
7439
7440	return rcStrict;
7441	}
7442
7443
7444	/**
7445	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7446	* VM-exit.
7447	*/
7448	HMVMX_EXIT_DECL vmxHCExitTripleFault(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7449	{
7450	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7451	return VINF_EM_RESET;
7452	}
7453
7454
7455	/**
7456	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7457	*/
7458	HMVMX_EXIT_DECL vmxHCExitHlt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7459	{
7460	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7461
7462	int rc = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
7463	AssertRCReturn(rc, rc);
7464
7465	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
7466	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
7467	rc = VINF_SUCCESS;
7468	else
7469	rc = VINF_EM_HALT;
7470
7471	if (rc != VINF_SUCCESS)
7472	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchHltToR3);
7473	return rc;
7474	}
7475
7476
7477	/**
7478	* VM-exit handler for instructions that result in a \#UD exception delivered to
7479	* the guest.
7480	*/
7481	HMVMX_EXIT_NSRC_DECL vmxHCExitSetPendingXcptUD(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7482	{
7483	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7484	vmxHCSetPendingXcptUD(pVCpu);
7485	return VINF_SUCCESS;
7486	}
7487
7488
7489	/**
7490	* VM-exit handler for expiry of the VMX-preemption timer.
7491	*/
7492	HMVMX_EXIT_DECL vmxHCExitPreemptTimer(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7493	{
7494	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7495
7496	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
7497	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
7498	Log12(("vmxHCExitPreemptTimer:\n"));
7499
7500	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7501	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7502	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7503	STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitPreemptTimer);
7504	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7505	}
7506
7507
7508	/**
7509	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7510	*/
7511	HMVMX_EXIT_DECL vmxHCExitXsetbv(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7512	{
7513	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7514
7515	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7516	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7517	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
7518	AssertRCReturn(rc, rc);
7519
7520	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbExitInstr);
7521	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
7522	: HM_CHANGED_RAISED_XCPT_MASK);
7523
7524	#ifndef IN_NEM_DARWIN
7525	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7526	bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
7527	if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
7528	{
7529	pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
7530	hmR0VmxUpdateStartVmFunction(pVCpu);
7531	}
7532	#endif
7533
7534	return rcStrict;
7535	}
7536
7537
7538	/**
7539	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7540	*/
7541	HMVMX_EXIT_DECL vmxHCExitInvpcid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7542	{
7543	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7544
7545	/** @todo Enable the new code after finding a reliably guest test-case. */
7546	#if 1
7547	return VERR_EM_INTERPRETER;
7548	#else
7549	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7550	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
7551	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
7552	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
7553	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
7554	AssertRCReturn(rc, rc);
7555
7556	/* Paranoia. Ensure this has a memory operand. */
7557	Assert(!pVmxTransient->ExitInstrInfo.Inv.u1Cleared0);
7558
7559	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
7560	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
7561	uint64_t const uType = CPUMIsGuestIn64BitCode(pVCpu) ? pVCpu->cpum.GstCtx.aGRegs[iGReg].u64
7562	: pVCpu->cpum.GstCtx.aGRegs[iGReg].u32;
7563
7564	RTGCPTR GCPtrDesc;
7565	HMVMX_DECODE_MEM_OPERAND(pVCpu, pVmxTransient->ExitInstrInfo.u, pVmxTransient->uExitQual, VMXMEMACCESS_READ, &GCPtrDesc);
7566
7567	VBOXSTRICTRC rcStrict = IEMExecDecodedInvpcid(pVCpu, pVmxTransient->cbExitInstr, pVmxTransient->ExitInstrInfo.Inv.iSegReg,
7568	GCPtrDesc, uType);
7569	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
7570	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
7571	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7572	{
7573	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7574	rcStrict = VINF_SUCCESS;
7575	}
7576	return rcStrict;
7577	#endif
7578	}
7579
7580
7581	/**
7582	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error
7583	* VM-exit.
7584	*/
7585	HMVMX_EXIT_NSRC_DECL vmxHCExitErrInvalidGuestState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7586	{
7587	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7588	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
7589	AssertRCReturn(rc, rc);
7590
7591	rc = vmxHCCheckCachedVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
7592	if (RT_FAILURE(rc))
7593	return rc;
7594
7595	uint32_t const uInvalidReason = vmxHCCheckGuestState(pVCpu, pVmcsInfo);
7596	NOREF(uInvalidReason);
7597
7598	#ifdef VBOX_STRICT
7599	uint32_t fIntrState;
7600	uint64_t u64Val;
7601	vmxHCReadEntryIntInfoVmcs(pVCpu, pVmxTransient);
7602	vmxHCReadEntryXcptErrorCodeVmcs(pVCpu, pVmxTransient);
7603	vmxHCReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
7604
7605	Log4(("uInvalidReason %u\n", uInvalidReason));
7606	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
7607	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7608	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7609
7610	rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &fIntrState); AssertRC(rc);
7611	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
7612	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR0, &u64Val); AssertRC(rc);
7613	Log4(("VMX_VMCS_GUEST_CR0 %#RX64\n", u64Val));
7614	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR0_MASK, &u64Val); AssertRC(rc);
7615	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RX64\n", u64Val));
7616	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Val); AssertRC(rc);
7617	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RX64\n", u64Val));
7618	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR4_MASK, &u64Val); AssertRC(rc);
7619	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RX64\n", u64Val));
7620	rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Val); AssertRC(rc);
7621	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RX64\n", u64Val));
7622	# ifndef IN_NEM_DARWIN
7623	if (pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
7624	{
7625	rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7626	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7627	}
7628
7629	hmR0DumpRegs(pVCpu, HM_DUMP_REG_FLAGS_ALL);
7630	# endif
7631	#endif
7632
7633	return VERR_VMX_INVALID_GUEST_STATE;
7634	}
7635
7636	/**
7637	* VM-exit handler for all undefined/unexpected reasons. Should never happen.
7638	*/
7639	HMVMX_EXIT_NSRC_DECL vmxHCExitErrUnexpected(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7640	{
7641	/*
7642	* Cumulative notes of all recognized but unexpected VM-exits.
7643	*
7644	* 1. This does -not- cover scenarios like a page-fault VM-exit occurring when
7645	* nested-paging is used.
7646	*
7647	* 2. Any instruction that causes a VM-exit unconditionally (for e.g. VMXON) must be
7648	* emulated or a #UD must be raised in the guest. Therefore, we should -not- be using
7649	* this function (and thereby stop VM execution) for handling such instructions.
7650	*
7651	*
7652	* VMX_EXIT_INIT_SIGNAL:
7653	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
7654	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these
7655	* VM-exits. However, we should not receive INIT signals VM-exit while executing a VM.
7656	*
7657	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery"
7658	* See Intel spec. 29.3 "VMX Instructions" for "VMXON".
7659	* See Intel spec. "23.8 Restrictions on VMX operation".
7660	*
7661	* VMX_EXIT_SIPI:
7662	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest
7663	* activity state is used. We don't make use of it as our guests don't have direct
7664	* access to the host local APIC.
7665	*
7666	* See Intel spec. 25.3 "Other Causes of VM-exits".
7667	*
7668	* VMX_EXIT_IO_SMI:
7669	* VMX_EXIT_SMI:
7670	* This can only happen if we support dual-monitor treatment of SMI, which can be
7671	* activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer
7672	* monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in
7673	* VMX root mode or receive an SMI. If we get here, something funny is going on.
7674	*
7675	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
7676	* See Intel spec. 25.3 "Other Causes of VM-Exits"
7677	*
7678	* VMX_EXIT_ERR_MSR_LOAD:
7679	* Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected
7680	* and typically indicates a bug in the hypervisor code. We thus cannot not resume
7681	* execution.
7682	*
7683	* See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State".
7684	*
7685	* VMX_EXIT_ERR_MACHINE_CHECK:
7686	* Machine check exceptions indicates a fatal/unrecoverable hardware condition
7687	* including but not limited to system bus, ECC, parity, cache and TLB errors. A
7688	* #MC exception abort class exception is raised. We thus cannot assume a
7689	* reasonable chance of continuing any sort of execution and we bail.
7690	*
7691	* See Intel spec. 15.1 "Machine-check Architecture".
7692	* See Intel spec. 27.1 "Architectural State Before A VM Exit".
7693	*
7694	* VMX_EXIT_PML_FULL:
7695	* VMX_EXIT_VIRTUALIZED_EOI:
7696	* VMX_EXIT_APIC_WRITE:
7697	* We do not currently support any of these features and thus they are all unexpected
7698	* VM-exits.
7699	*
7700	* VMX_EXIT_GDTR_IDTR_ACCESS:
7701	* VMX_EXIT_LDTR_TR_ACCESS:
7702	* VMX_EXIT_RDRAND:
7703	* VMX_EXIT_RSM:
7704	* VMX_EXIT_VMFUNC:
7705	* VMX_EXIT_ENCLS:
7706	* VMX_EXIT_RDSEED:
7707	* VMX_EXIT_XSAVES:
7708	* VMX_EXIT_XRSTORS:
7709	* VMX_EXIT_UMWAIT:
7710	* VMX_EXIT_TPAUSE:
7711	* VMX_EXIT_LOADIWKEY:
7712	* These VM-exits are -not- caused unconditionally by execution of the corresponding
7713	* instruction. Any VM-exit for these instructions indicate a hardware problem,
7714	* unsupported CPU modes (like SMM) or potentially corrupt VMCS controls.
7715	*
7716	* See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
7717	*/
7718	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7719	AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason));
7720	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
7721	}
7722
7723
7724	/**
7725	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7726	*/
7727	HMVMX_EXIT_DECL vmxHCExitRdmsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7728	{
7729	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7730
7731	/** @todo Optimize this: We currently drag in the whole MSR state
7732	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
7733	* MSRs required. That would require changes to IEM and possibly CPUM too.
7734	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
7735	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7736	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
7737	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
7738	switch (idMsr)
7739	{
7740	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
7741	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
7742	}
7743
7744	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7745	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, fImport);
7746	AssertRCReturn(rc, rc);
7747
7748	Log4Func(("ecx=%#RX32\n", idMsr));
7749
7750	#if defined(VBOX_STRICT) && !defined(IN_NEM_DARWIN)
7751	Assert(!pVmxTransient->fIsNestedGuest);
7752	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
7753	{
7754	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
7755	&& idMsr != MSR_K6_EFER)
7756	{
7757	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
7758	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
7759	}
7760	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
7761	{
7762	Assert(pVmcsInfo->pvMsrBitmap);
7763	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
7764	if (fMsrpm & VMXMSRPM_ALLOW_RD)
7765	{
7766	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
7767	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
7768	}
7769	}
7770	}
7771	#endif
7772
7773	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbExitInstr);
7774	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitRdmsr);
7775	if (rcStrict == VINF_SUCCESS)
7776	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
7777	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7778	{
7779	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7780	rcStrict = VINF_SUCCESS;
7781	}
7782	else
7783	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ \|\| rcStrict == VINF_EM_TRIPLE_FAULT,
7784	("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
7785
7786	return rcStrict;
7787	}
7788
7789
7790	/**
7791	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7792	*/
7793	HMVMX_EXIT_DECL vmxHCExitWrmsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7794	{
7795	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7796
7797	/** @todo Optimize this: We currently drag in the whole MSR state
7798	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
7799	* MSRs required. That would require changes to IEM and possibly CPUM too.
7800	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
7801	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
7802	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
7803
7804	/*
7805	* The FS and GS base MSRs are not part of the above all-MSRs mask.
7806	* Although we don't need to fetch the base as it will be overwritten shortly, while
7807	* loading guest-state we would also load the entire segment register including limit
7808	* and attributes and thus we need to load them here.
7809	*/
7810	switch (idMsr)
7811	{
7812	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
7813	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
7814	}
7815
7816	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7817	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7818	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, fImport);
7819	AssertRCReturn(rc, rc);
7820
7821	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
7822
7823	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbExitInstr);
7824	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitWrmsr);
7825
7826	if (rcStrict == VINF_SUCCESS)
7827	{
7828	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
7829
7830	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7831	if ( idMsr == MSR_IA32_APICBASE
7832	\|\| ( idMsr >= MSR_IA32_X2APIC_START
7833	&& idMsr <= MSR_IA32_X2APIC_END))
7834	{
7835	/*
7836	* We've already saved the APIC related guest-state (TPR) in post-run phase.
7837	* When full APIC register virtualization is implemented we'll have to make
7838	* sure APIC state is saved from the VMCS before IEM changes it.
7839	*/
7840	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7841	}
7842	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
7843	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
7844	else if (idMsr == MSR_K6_EFER)
7845	{
7846	/*
7847	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
7848	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
7849	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
7850	*/
7851	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
7852	}
7853
7854	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not used. */
7855	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
7856	{
7857	switch (idMsr)
7858	{
7859	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
7860	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
7861	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
7862	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_FS); break;
7863	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_GS); break;
7864	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
7865	default:
7866	{
7867	#ifndef IN_NEM_DARWIN
7868	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
7869	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
7870	else if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
7871	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
7872	#else
7873	AssertMsgFailed(("TODO\n"));
7874	#endif
7875	break;
7876	}
7877	}
7878	}
7879	#if defined(VBOX_STRICT) && !defined(IN_NEM_DARWIN)
7880	else
7881	{
7882	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7883	switch (idMsr)
7884	{
7885	case MSR_IA32_SYSENTER_CS:
7886	case MSR_IA32_SYSENTER_EIP:
7887	case MSR_IA32_SYSENTER_ESP:
7888	case MSR_K8_FS_BASE:
7889	case MSR_K8_GS_BASE:
7890	{
7891	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
7892	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
7893	}
7894
7895	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
7896	default:
7897	{
7898	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
7899	{
7900	/* EFER MSR writes are always intercepted. */
7901	if (idMsr != MSR_K6_EFER)
7902	{
7903	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
7904	idMsr));
7905	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
7906	}
7907	}
7908
7909	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
7910	{
7911	Assert(pVmcsInfo->pvMsrBitmap);
7912	uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
7913	if (fMsrpm & VMXMSRPM_ALLOW_WR)
7914	{
7915	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
7916	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
7917	}
7918	}
7919	break;
7920	}
7921	}
7922	}
7923	#endif /* VBOX_STRICT */
7924	}
7925	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7926	{
7927	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7928	rcStrict = VINF_SUCCESS;
7929	}
7930	else
7931	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE \|\| rcStrict == VINF_EM_TRIPLE_FAULT,
7932	("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
7933
7934	return rcStrict;
7935	}
7936
7937
7938	/**
7939	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
7940	*/
7941	HMVMX_EXIT_DECL vmxHCExitPause(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7942	{
7943	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7944
7945	/** @todo The guest has likely hit a contended spinlock. We might want to
7946	* poke a schedule different guest VCPU. */
7947	int rc = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
7948	if (RT_SUCCESS(rc))
7949	return VINF_EM_RAW_INTERRUPT;
7950
7951	AssertMsgFailed(("vmxHCExitPause: Failed to increment RIP. rc=%Rrc\n", rc));
7952	return rc;
7953	}
7954
7955
7956	/**
7957	* VM-exit handler for when the TPR value is lowered below the specified
7958	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
7959	*/
7960	HMVMX_EXIT_NSRC_DECL vmxHCExitTprBelowThreshold(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7961	{
7962	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7963	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
7964
7965	/*
7966	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
7967	* We'll re-evaluate pending interrupts and inject them before the next VM
7968	* entry so we can just continue execution here.
7969	*/
7970	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitTprBelowThreshold);
7971	return VINF_SUCCESS;
7972	}
7973
7974
7975	/**
7976	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
7977	* VM-exit.
7978	*
7979	* @retval VINF_SUCCESS when guest execution can continue.
7980	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
7981	* @retval VERR_EM_RESCHEDULE_REM when we need to return to ring-3 due to
7982	* incompatible guest state for VMX execution (real-on-v86 case).
7983	*/
7984	HMVMX_EXIT_DECL vmxHCExitMovCRx(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
7985	{
7986	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
7987	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitMovCRx, y2);
7988
7989	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
7990	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
7991	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7992
7993	VBOXSTRICTRC rcStrict;
7994	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7995	uint64_t const uExitQual = pVmxTransient->uExitQual;
7996	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
7997	switch (uAccessType)
7998	{
7999	/*
8000	* MOV to CRx.
8001	*/
8002	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
8003	{
8004	/*
8005	* When PAE paging is used, the CPU will reload PAE PDPTEs from CR3 when the guest
8006	* changes certain bits even in CR0, CR4 (and not just CR3). We are currently fine
8007	* since IEM_CPUMCTX_EXTRN_MUST_MASK (used below) includes CR3 which will import
8008	* PAE PDPTEs as well.
8009	*/
8010	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
8011	AssertRCReturn(rc, rc);
8012
8013	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
8014	#ifndef IN_NEM_DARWIN
8015	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
8016	#endif
8017	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
8018	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
8019
8020	/*
8021	* MOV to CR3 only cause a VM-exit when one or more of the following are true:
8022	* - When nested paging isn't used.
8023	* - If the guest doesn't have paging enabled (intercept CR3 to update shadow page tables).
8024	* - We are executing in the VM debug loop.
8025	*/
8026	#ifndef HMVMX_ALAWAYS_INTERCEPT_CR3_ACCESS
8027	# ifndef IN_NEM_DARWIN
8028	Assert( iCrReg != 3
8029	\|\| !VM_IS_VMX_NESTED_PAGING(pVM)
8030	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
8031	\|\| pVCpu->hmr0.s.fUsingDebugLoop);
8032	# else
8033	Assert( iCrReg != 3
8034	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx));
8035	# endif
8036	#endif
8037
8038	/* MOV to CR8 writes only cause VM-exits when TPR shadow is not used. */
8039	Assert( iCrReg != 8
8040	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
8041
8042	rcStrict = vmxHCExitMovToCrX(pVCpu, pVmxTransient->cbExitInstr, iGReg, iCrReg);
8043	AssertMsg( rcStrict == VINF_SUCCESS
8044	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8045
8046	#ifndef IN_NEM_DARWIN
8047	/*
8048	* This is a kludge for handling switches back to real mode when we try to use
8049	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
8050	* deal with special selector values, so we have to return to ring-3 and run
8051	* there till the selector values are V86 mode compatible.
8052	*
8053	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
8054	* latter is an alias for VINF_IEM_RAISED_XCPT which is asserted at the end of
8055	* this function.
8056	*/
8057	if ( iCrReg == 0
8058	&& rcStrict == VINF_SUCCESS
8059	&& !VM_IS_VMX_UNRESTRICTED_GUEST(pVM)
8060	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
8061	&& (uOldCr0 & X86_CR0_PE)
8062	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
8063	{
8064	/** @todo Check selectors rather than returning all the time. */
8065	Assert(!pVmxTransient->fIsNestedGuest);
8066	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
8067	rcStrict = VINF_EM_RESCHEDULE_REM;
8068	}
8069	#endif
8070
8071	break;
8072	}
8073
8074	/*
8075	* MOV from CRx.
8076	*/
8077	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
8078	{
8079	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
8080	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
8081
8082	/*
8083	* MOV from CR3 only cause a VM-exit when one or more of the following are true:
8084	* - When nested paging isn't used.
8085	* - If the guest doesn't have paging enabled (pass guest's CR3 rather than our identity mapped CR3).
8086	* - We are executing in the VM debug loop.
8087	*/
8088	#ifndef HMVMX_ALAWAYS_INTERCEPT_CR3_ACCESS
8089	# ifndef IN_NEM_DARWIN
8090	Assert( iCrReg != 3
8091	\|\| !VM_IS_VMX_NESTED_PAGING(pVM)
8092	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
8093	\|\| pVCpu->hmr0.s.fLeaveDone);
8094	# else
8095	Assert( iCrReg != 3
8096	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx));
8097	# endif
8098	#endif
8099
8100	/* MOV from CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
8101	Assert( iCrReg != 8
8102	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
8103
8104	rcStrict = vmxHCExitMovFromCrX(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
8105	break;
8106	}
8107
8108	/*
8109	* CLTS (Clear Task-Switch Flag in CR0).
8110	*/
8111	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
8112	{
8113	rcStrict = vmxHCExitClts(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr);
8114	break;
8115	}
8116
8117	/*
8118	* LMSW (Load Machine-Status Word into CR0).
8119	* LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here.
8120	*/
8121	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW:
8122	{
8123	RTGCPTR GCPtrEffDst;
8124	uint8_t const cbInstr = pVmxTransient->cbExitInstr;
8125	uint16_t const uMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual);
8126	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(uExitQual);
8127	if (fMemOperand)
8128	{
8129	vmxHCReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
8130	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
8131	}
8132	else
8133	GCPtrEffDst = NIL_RTGCPTR;
8134	rcStrict = vmxHCExitLmsw(pVCpu, pVmcsInfo, cbInstr, uMsw, GCPtrEffDst);
8135	break;
8136	}
8137
8138	default:
8139	{
8140	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
8141	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
8142	}
8143	}
8144
8145	Assert((VCPU_2_VMXSTATE(pVCpu).fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
8146	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
8147	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
8148
8149	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitMovCRx, y2);
8150	NOREF(pVM);
8151	return rcStrict;
8152	}
8153
8154
8155	/**
8156	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
8157	* VM-exit.
8158	*/
8159	HMVMX_EXIT_DECL vmxHCExitIoInstr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8160	{
8161	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8162	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitIO, y1);
8163
8164	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8165	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8166	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
8167	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8168	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
8169	\| CPUMCTX_EXTRN_EFER);
8170	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
8171	AssertRCReturn(rc, rc);
8172
8173	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
8174	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
8175	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
8176	bool const fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
8177	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
8178	bool const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
8179	bool const fDbgStepping = VCPU_2_VMXSTATE(pVCpu).fSingleInstruction;
8180	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
8181
8182	/*
8183	* Update exit history to see if this exit can be optimized.
8184	*/
8185	VBOXSTRICTRC rcStrict;
8186	PCEMEXITREC pExitRec = NULL;
8187	if ( !fGstStepping
8188	&& !fDbgStepping)
8189	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
8190	!fIOString
8191	? !fIOWrite
8192	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
8193	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
8194	: !fIOWrite
8195	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
8196	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
8197	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
8198	if (!pExitRec)
8199	{
8200	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
8201	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
8202
8203	uint32_t const cbValue = s_aIOSizes[uIOSize];
8204	uint32_t const cbInstr = pVmxTransient->cbExitInstr;
8205	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
8206	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8207	if (fIOString)
8208	{
8209	/*
8210	* INS/OUTS - I/O String instruction.
8211	*
8212	* Use instruction-information if available, otherwise fall back on
8213	* interpreting the instruction.
8214	*/
8215	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
8216	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
8217	bool const fInsOutsInfo = RT_BF_GET(g_HmMsrs.u.vmx.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
8218	if (fInsOutsInfo)
8219	{
8220	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
8221	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
8222	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
8223	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
8224	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
8225	if (fIOWrite)
8226	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
8227	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
8228	else
8229	{
8230	/*
8231	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
8232	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
8233	* See Intel Instruction spec. for "INS".
8234	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
8235	*/
8236	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
8237	}
8238	}
8239	else
8240	rcStrict = IEMExecOne(pVCpu);
8241
8242	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP);
8243	fUpdateRipAlready = true;
8244	}
8245	else
8246	{
8247	/*
8248	* IN/OUT - I/O instruction.
8249	*/
8250	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
8251	uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
8252	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
8253	if (fIOWrite)
8254	{
8255	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
8256	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitIOWrite);
8257	#ifndef IN_NEM_DARWIN
8258	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
8259	&& !pCtx->eflags.Bits.u1TF)
8260	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
8261	#endif
8262	}
8263	else
8264	{
8265	uint32_t u32Result = 0;
8266	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
8267	if (IOM_SUCCESS(rcStrict))
8268	{
8269	/* Save result of I/O IN instr. in AL/AX/EAX. */
8270	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
8271	}
8272	#ifndef IN_NEM_DARWIN
8273	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
8274	&& !pCtx->eflags.Bits.u1TF)
8275	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
8276	#endif
8277	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitIORead);
8278	}
8279	}
8280
8281	if (IOM_SUCCESS(rcStrict))
8282	{
8283	if (!fUpdateRipAlready)
8284	{
8285	vmxHCAdvanceGuestRipBy(pVCpu, cbInstr);
8286	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP);
8287	}
8288
8289	/*
8290	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
8291	* while booting Fedora 17 64-bit guest.
8292	*
8293	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
8294	*/
8295	if (fIOString)
8296	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
8297
8298	/*
8299	* If any I/O breakpoints are armed, we need to check if one triggered
8300	* and take appropriate action.
8301	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
8302	*/
8303	rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
8304	AssertRCReturn(rc, rc);
8305
8306	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
8307	* execution engines about whether hyper BPs and such are pending. */
8308	uint32_t const uDr7 = pCtx->dr[7];
8309	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
8310	&& X86_DR7_ANY_RW_IO(uDr7)
8311	&& (pCtx->cr4 & X86_CR4_DE))
8312	\|\| DBGFBpIsHwIoArmed(pVM)))
8313	{
8314	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatDRxIoCheck);
8315
8316	#ifndef IN_NEM_DARWIN
8317	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
8318	VMMRZCallRing3Disable(pVCpu);
8319	HM_DISABLE_PREEMPT(pVCpu);
8320
8321	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
8322
8323	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
8324	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
8325	{
8326	/* Raise #DB. */
8327	if (fIsGuestDbgActive)
8328	ASMSetDR6(pCtx->dr[6]);
8329	if (pCtx->dr[7] != uDr7)
8330	VCPU_2_VMXSTATE(pVCpu).fCtxChanged \|= HM_CHANGED_GUEST_DR7;
8331
8332	vmxHCSetPendingXcptDB(pVCpu);
8333	}
8334	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
8335	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
8336	else if ( rcStrict2 != VINF_SUCCESS
8337	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
8338	rcStrict = rcStrict2;
8339	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
8340
8341	HM_RESTORE_PREEMPT();
8342	VMMRZCallRing3Enable(pVCpu);
8343	#else
8344	/** @todo */
8345	#endif
8346	}
8347	}
8348
8349	#ifdef VBOX_STRICT
8350	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
8351	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
8352	Assert(!fIOWrite);
8353	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
8354	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
8355	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
8356	Assert(fIOWrite);
8357	else
8358	{
8359	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
8360	* statuses, that the VMM device and some others may return. See
8361	* IOM_SUCCESS() for guidance. */
8362	AssertMsg( RT_FAILURE(rcStrict)
8363	\|\| rcStrict == VINF_SUCCESS
8364	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
8365	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
8366	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
8367	\|\| rcStrict == VINF_EM_RAW_TO_R3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8368	# endif
8369	}
8370	#endif
8371	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitIO, y1);
8372	}
8373	else
8374	{
8375	/*
8376	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
8377	*/
8378	int rc2 = vmxHCImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8379	AssertRCReturn(rc2, rc2);
8380	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &VCPU_2_VMXSTATS(pVCpu).StatExitIOWrite : &VCPU_2_VMXSTATS(pVCpu).StatExitIORead
8381	: fIOWrite ? &VCPU_2_VMXSTATS(pVCpu).StatExitIOStringWrite : &VCPU_2_VMXSTATS(pVCpu).StatExitIOStringRead);
8382	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
8383	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
8384	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
8385	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
8386
8387	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
8388	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
8389
8390	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
8391	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
8392	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
8393	}
8394	return rcStrict;
8395	}
8396
8397
8398	/**
8399	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
8400	* VM-exit.
8401	*/
8402	HMVMX_EXIT_DECL vmxHCExitTaskSwitch(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8403	{
8404	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8405
8406	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
8407	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
8408	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
8409	{
8410	vmxHCReadIdtVectoringInfoVmcs(pVCpu, pVmxTransient);
8411	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
8412	{
8413	uint32_t uErrCode;
8414	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
8415	{
8416	vmxHCReadIdtVectoringErrorCodeVmcs(pVCpu, pVmxTransient);
8417	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
8418	}
8419	else
8420	uErrCode = 0;
8421
8422	RTGCUINTPTR GCPtrFaultAddress;
8423	if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(pVmxTransient->uIdtVectoringInfo))
8424	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
8425	else
8426	GCPtrFaultAddress = 0;
8427
8428	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8429
8430	vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
8431	pVmxTransient->cbExitInstr, uErrCode, GCPtrFaultAddress);
8432
8433	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo),
8434	VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo)));
8435	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitTaskSwitch);
8436	return VINF_EM_RAW_INJECT_TRPM_EVENT;
8437	}
8438	}
8439
8440	/* Fall back to the interpreter to emulate the task-switch. */
8441	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitTaskSwitch);
8442	return VERR_EM_INTERPRETER;
8443	}
8444
8445
8446	/**
8447	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
8448	*/
8449	HMVMX_EXIT_DECL vmxHCExitMtf(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8450	{
8451	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8452
8453	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8454	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
8455	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8456	AssertRC(rc);
8457	return VINF_EM_DBG_STEPPED;
8458	}
8459
8460
8461	/**
8462	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
8463	*/
8464	HMVMX_EXIT_DECL vmxHCExitApicAccess(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8465	{
8466	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8467	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitApicAccess);
8468
8469	vmxHCReadExitIntInfoVmcs(pVCpu, pVmxTransient);
8470	vmxHCReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
8471	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8472	vmxHCReadIdtVectoringInfoVmcs(pVCpu, pVmxTransient);
8473	vmxHCReadIdtVectoringErrorCodeVmcs(pVCpu, pVmxTransient);
8474
8475	/*
8476	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
8477	*/
8478	VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
8479	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8480	{
8481	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
8482	if (RT_UNLIKELY(VCPU_2_VMXSTATE(pVCpu).Event.fPending))
8483	{
8484	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectInterpret);
8485	return VINF_EM_RAW_INJECT_TRPM_EVENT;
8486	}
8487	}
8488	else
8489	{
8490	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
8491	return rcStrict;
8492	}
8493
8494	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
8495	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8496	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
8497	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
8498	AssertRCReturn(rc, rc);
8499
8500	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8501	uint32_t const uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
8502	switch (uAccessType)
8503	{
8504	#ifndef IN_NEM_DARWIN
8505	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8506	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8507	{
8508	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
8509	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
8510	("vmxHCExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
8511
8512	RTGCPHYS GCPhys = VCPU_2_VMXSTATE(pVCpu).vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
8513	GCPhys &= PAGE_BASE_GC_MASK;
8514	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
8515	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
8516	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
8517
8518	rcStrict = IOMR0MmioPhysHandler(pVCpu->CTX_SUFF(pVM), pVCpu,
8519	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW, GCPhys);
8520	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8521	if ( rcStrict == VINF_SUCCESS
8522	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
8523	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
8524	{
8525	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8526	\| HM_CHANGED_GUEST_APIC_TPR);
8527	rcStrict = VINF_SUCCESS;
8528	}
8529	break;
8530	}
8531	#else
8532	/** @todo */
8533	#endif
8534
8535	default:
8536	{
8537	Log4Func(("uAccessType=%#x\n", uAccessType));
8538	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
8539	break;
8540	}
8541	}
8542
8543	if (rcStrict != VINF_SUCCESS)
8544	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchApicAccessToR3);
8545	return rcStrict;
8546	}
8547
8548
8549	/**
8550	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8551	* VM-exit.
8552	*/
8553	HMVMX_EXIT_DECL vmxHCExitMovDRx(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8554	{
8555	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8556	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8557
8558	/*
8559	* We might also get this VM-exit if the nested-guest isn't intercepting MOV DRx accesses.
8560	* In such a case, rather than disabling MOV DRx intercepts and resuming execution, we
8561	* must emulate the MOV DRx access.
8562	*/
8563	if (!pVmxTransient->fIsNestedGuest)
8564	{
8565	/* We should -not- get this VM-exit if the guest's debug registers were active. */
8566	if (pVmxTransient->fWasGuestDebugStateActive)
8567	{
8568	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
8569	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
8570	}
8571
8572	if ( !VCPU_2_VMXSTATE(pVCpu).fSingleInstruction
8573	&& !pVmxTransient->fWasHyperDebugStateActive)
8574	{
8575	Assert(!DBGFIsStepping(pVCpu));
8576	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
8577
8578	/* Don't intercept MOV DRx any more. */
8579	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
8580	int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
8581	AssertRC(rc);
8582
8583	#ifndef IN_NEM_DARWIN
8584	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
8585	VMMRZCallRing3Disable(pVCpu);
8586	HM_DISABLE_PREEMPT(pVCpu);
8587
8588	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8589	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
8590	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8591
8592	HM_RESTORE_PREEMPT();
8593	VMMRZCallRing3Enable(pVCpu);
8594	#else
8595	/** @todo */
8596	#endif
8597
8598	#ifdef VBOX_WITH_STATISTICS
8599	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
8600	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
8601	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitDRxWrite);
8602	else
8603	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitDRxRead);
8604	#endif
8605	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatDRxContextSwitch);
8606	return VINF_SUCCESS;
8607	}
8608	}
8609
8610	/*
8611	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
8612	* The EFER MSR is always up-to-date.
8613	* Update the segment registers and DR7 from the CPU.
8614	*/
8615	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8616	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
8617	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
8618	AssertRCReturn(rc, rc);
8619	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
8620
8621	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8622	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
8623	{
8624	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
8625	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
8626	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
8627	if (RT_SUCCESS(rc))
8628	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_DR7);
8629	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitDRxWrite);
8630	}
8631	else
8632	{
8633	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
8634	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
8635	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
8636	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitDRxRead);
8637	}
8638
8639	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8640	if (RT_SUCCESS(rc))
8641	{
8642	int rc2 = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
8643	AssertRCReturn(rc2, rc2);
8644	return VINF_SUCCESS;
8645	}
8646	return rc;
8647	}
8648
8649
8650	/**
8651	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8652	* Conditional VM-exit.
8653	*/
8654	HMVMX_EXIT_DECL vmxHCExitEptMisconfig(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8655	{
8656	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8657
8658	#ifndef IN_NEM_DARWIN
8659	Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
8660
8661	vmxHCReadExitIntInfoVmcs(pVCpu, pVmxTransient);
8662	vmxHCReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
8663	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8664	vmxHCReadIdtVectoringInfoVmcs(pVCpu, pVmxTransient);
8665	vmxHCReadIdtVectoringErrorCodeVmcs(pVCpu, pVmxTransient);
8666
8667	/*
8668	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
8669	*/
8670	VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
8671	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8672	{
8673	/*
8674	* In the unlikely case where delivering an event causes an EPT misconfig (MMIO), go back to
8675	* instruction emulation to inject the original event. Otherwise, injecting the original event
8676	* using hardware-assisted VMX would trigger the same EPT misconfig VM-exit again.
8677	*/
8678	if (!VCPU_2_VMXSTATE(pVCpu).Event.fPending)
8679	{ /* likely */ }
8680	else
8681	{
8682	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectInterpret);
8683	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8684	/** @todo NSTVMX: Think about how this should be handled. */
8685	if (pVmxTransient->fIsNestedGuest)
8686	return VERR_VMX_IPE_3;
8687	#endif
8688	return VINF_EM_RAW_INJECT_TRPM_EVENT;
8689	}
8690	}
8691	else
8692	{
8693	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
8694	return rcStrict;
8695	}
8696
8697	/*
8698	* Get sufficient state and update the exit history entry.
8699	*/
8700	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8701	vmxHCReadGuestPhysicalAddrVmcs(pVCpu, pVmxTransient);
8702	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
8703	AssertRCReturn(rc, rc);
8704
8705	RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr;
8706	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
8707	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
8708	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
8709	if (!pExitRec)
8710	{
8711	/*
8712	* If we succeed, resume guest execution.
8713	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8714	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8715	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8716	* weird case. See @bugref{6043}.
8717	*/
8718	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8719	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8720	/** @todo bird: We can probably just go straight to IOM here and assume that
8721	* it's MMIO, then fall back on PGM if that hunch didn't work out so
8722	* well. However, we need to address that aliasing workarounds that
8723	* PGMR0Trap0eHandlerNPMisconfig implements. So, some care is needed.
8724	*
8725	* Might also be interesting to see if we can get this done more or
8726	* less locklessly inside IOM. Need to consider the lookup table
8727	* updating and use a bit more carefully first (or do all updates via
8728	* rendezvous) */
8729	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
8730	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
8731	if ( rcStrict == VINF_SUCCESS
8732	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
8733	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
8734	{
8735	/* Successfully handled MMIO operation. */
8736	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8737	\| HM_CHANGED_GUEST_APIC_TPR);
8738	rcStrict = VINF_SUCCESS;
8739	}
8740	}
8741	else
8742	{
8743	/*
8744	* Frequent exit or something needing probing. Call EMHistoryExec.
8745	*/
8746	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
8747	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
8748
8749	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
8750	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
8751
8752	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
8753	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
8754	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
8755	}
8756	return rcStrict;
8757	#else
8758	AssertFailed();
8759	return VERR_VMX_IPE_3; /* Should never happen with Apple HV in R3. */
8760	#endif
8761	}
8762
8763
8764	/**
8765	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8766	* VM-exit.
8767	*/
8768	HMVMX_EXIT_DECL vmxHCExitEptViolation(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8769	{
8770	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8771	#ifndef IN_NEM_DARWIN
8772	Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
8773
8774	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
8775	vmxHCReadExitIntInfoVmcs(pVCpu, pVmxTransient);
8776	vmxHCReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
8777	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8778	vmxHCReadIdtVectoringInfoVmcs(pVCpu, pVmxTransient);
8779	vmxHCReadIdtVectoringErrorCodeVmcs(pVCpu, pVmxTransient);
8780
8781	/*
8782	* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
8783	*/
8784	VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
8785	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8786	{
8787	/*
8788	* If delivery of an event causes an EPT violation (true nested #PF and not MMIO),
8789	* we shall resolve the nested #PF and re-inject the original event.
8790	*/
8791	if (VCPU_2_VMXSTATE(pVCpu).Event.fPending)
8792	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectReflectNPF);
8793	}
8794	else
8795	{
8796	Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
8797	return rcStrict;
8798	}
8799
8800	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8801	vmxHCReadGuestPhysicalAddrVmcs(pVCpu, pVmxTransient);
8802	int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
8803	AssertRCReturn(rc, rc);
8804
8805	RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr;
8806	uint64_t const uExitQual = pVmxTransient->uExitQual;
8807	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", uExitQual));
8808
8809	RTGCUINT uErrorCode = 0;
8810	if (uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_INSTR_FETCH)
8811	uErrorCode \|= X86_TRAP_PF_ID;
8812	if (uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE)
8813	uErrorCode \|= X86_TRAP_PF_RW;
8814	if (uExitQual & (VMX_EXIT_QUAL_EPT_ENTRY_READ \| VMX_EXIT_QUAL_EPT_ENTRY_WRITE \| VMX_EXIT_QUAL_EPT_ENTRY_EXECUTE))
8815	uErrorCode \|= X86_TRAP_PF_P;
8816
8817	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8818	Log4Func(("at %#RX64 (%#RX64 errcode=%#x) cs:rip=%#04x:%#RX64\n", GCPhys, uExitQual, uErrorCode, pCtx->cs.Sel, pCtx->rip));
8819
8820	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8821
8822	/*
8823	* Handle the pagefault trap for the nested shadow table.
8824	*/
8825	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
8826	rcStrict = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
8827	TRPMResetTrap(pVCpu);
8828
8829	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8830	if ( rcStrict == VINF_SUCCESS
8831	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
8832	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
8833	{
8834	/* Successfully synced our nested page tables. */
8835	STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitReasonNpf);
8836	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
8837	return VINF_SUCCESS;
8838	}
8839	#else
8840	PVM pVM = pVCpu->CTX_SUFF(pVM);
8841	uint64_t const uHostTsc = ASMReadTSC(); RT_NOREF(uHostTsc);
8842	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
8843	vmxHCReadGuestPhysicalAddrVmcs(pVCpu, pVmxTransient);
8844	vmxHCImportGuestRip(pVCpu);
8845	vmxHCImportGuestSegReg(pVCpu, X86_SREG_CS);
8846
8847	/*
8848	* Ask PGM for information about the given GCPhys. We need to check if we're
8849	* out of sync first.
8850	*/
8851	NEMHCDARWINHMACPCCSTATE State = { RT_BOOL(pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE), false, false };
8852	PGMPHYSNEMPAGEINFO Info;
8853	int rc = PGMPhysNemPageInfoChecker(pVM, pVCpu, pVmxTransient->uGuestPhysicalAddr, State.fWriteAccess, &Info,
8854	nemR3DarwinHandleMemoryAccessPageCheckerCallback, &State);
8855	if (RT_SUCCESS(rc))
8856	{
8857	if (Info.fNemProt & ( RT_BOOL(pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE)
8858	? NEM_PAGE_PROT_WRITE : NEM_PAGE_PROT_READ))
8859	{
8860	if (State.fCanResume)
8861	{
8862	Log4(("MemExit/%u: %04x:%08RX64: %RGp (=>%RHp) %s fProt=%u%s%s%s; restarting\n",
8863	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
8864	pVmxTransient->uGuestPhysicalAddr, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
8865	Info.fHasHandlers ? " handlers" : "", Info.fZeroPage ? " zero-pg" : "",
8866	State.fDidSomething ? "" : " no-change"));
8867	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_MEMORY_ACCESS),
8868	pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, uHostTsc);
8869	return VINF_SUCCESS;
8870	}
8871	}
8872
8873	Log4(("MemExit/%u: %04x:%08RX64: %RGp (=>%RHp) %s fProt=%u%s%s%s; emulating\n",
8874	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
8875	pVmxTransient->uGuestPhysicalAddr, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
8876	Info.fHasHandlers ? " handlers" : "", Info.fZeroPage ? " zero-pg" : "",
8877	State.fDidSomething ? "" : " no-change"));
8878	}
8879	else
8880	Log4(("MemExit/%u: %04x:%08RX64: %RGp rc=%Rrc%s; emulating\n",
8881	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
8882	pVmxTransient->uGuestPhysicalAddr, rc, State.fDidSomething ? " modified-backing" : ""));
8883
8884	/*
8885	* Emulate the memory access, either access handler or special memory.
8886	*/
8887	PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
8888	RT_BOOL(pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE)
8889	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_WRITE)
8890	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_READ),
8891	pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, uHostTsc);
8892
8893	rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8894	AssertRCReturn(rc, rc);
8895
8896	VBOXSTRICTRC rcStrict;
8897	if (!pExitRec)
8898	rcStrict = IEMExecOne(pVCpu);
8899	else
8900	{
8901	/* Frequent access or probing. */
8902	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
8903	Log4(("MemExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
8904	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
8905	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
8906	}
8907
8908	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
8909	#endif
8910
8911	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8912	return rcStrict;
8913	}
8914
8915
8916	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8917	/**
8918	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
8919	*/
8920	HMVMX_EXIT_DECL vmxHCExitVmclear(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8921	{
8922	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8923
8924	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8925	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
8926	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
8927	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
8928	\| CPUMCTX_EXTRN_HWVIRT
8929	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
8930	AssertRCReturn(rc, rc);
8931
8932	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
8933
8934	VMXVEXITINFO ExitInfo;
8935	RT_ZERO(ExitInfo);
8936	ExitInfo.uReason = pVmxTransient->uExitReason;
8937	ExitInfo.u64Qual = pVmxTransient->uExitQual;
8938	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
8939	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
8940	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
8941
8942	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
8943	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8944	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
8945	else if (rcStrict == VINF_IEM_RAISED_XCPT)
8946	{
8947	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
8948	rcStrict = VINF_SUCCESS;
8949	}
8950	return rcStrict;
8951	}
8952
8953
8954	/**
8955	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
8956	*/
8957	HMVMX_EXIT_DECL vmxHCExitVmlaunch(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8958	{
8959	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8960
8961	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
8962	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
8963	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8964	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8965	AssertRCReturn(rc, rc);
8966
8967	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
8968
8969	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitVmentry, z);
8970	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbExitInstr, VMXINSTRID_VMLAUNCH);
8971	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitVmentry, z);
8972	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8973	{
8974	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
8975	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8976	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
8977	}
8978	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
8979	return rcStrict;
8980	}
8981
8982
8983	/**
8984	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
8985	*/
8986	HMVMX_EXIT_DECL vmxHCExitVmptrld(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
8987	{
8988	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
8989
8990	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8991	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
8992	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
8993	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
8994	\| CPUMCTX_EXTRN_HWVIRT
8995	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
8996	AssertRCReturn(rc, rc);
8997
8998	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
8999
9000	VMXVEXITINFO ExitInfo;
9001	RT_ZERO(ExitInfo);
9002	ExitInfo.uReason = pVmxTransient->uExitReason;
9003	ExitInfo.u64Qual = pVmxTransient->uExitQual;
9004	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
9005	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
9006	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
9007
9008	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
9009	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9010	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
9011	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9012	{
9013	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9014	rcStrict = VINF_SUCCESS;
9015	}
9016	return rcStrict;
9017	}
9018
9019
9020	/**
9021	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
9022	*/
9023	HMVMX_EXIT_DECL vmxHCExitVmptrst(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9024	{
9025	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9026
9027	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9028	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
9029	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
9030	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
9031	\| CPUMCTX_EXTRN_HWVIRT
9032	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
9033	AssertRCReturn(rc, rc);
9034
9035	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
9036
9037	VMXVEXITINFO ExitInfo;
9038	RT_ZERO(ExitInfo);
9039	ExitInfo.uReason = pVmxTransient->uExitReason;
9040	ExitInfo.u64Qual = pVmxTransient->uExitQual;
9041	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
9042	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
9043	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
9044
9045	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
9046	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9047	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
9048	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9049	{
9050	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9051	rcStrict = VINF_SUCCESS;
9052	}
9053	return rcStrict;
9054	}
9055
9056
9057	/**
9058	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Conditional VM-exit.
9059	*/
9060	HMVMX_EXIT_DECL vmxHCExitVmread(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9061	{
9062	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9063
9064	/*
9065	* Strictly speaking we should not get VMREAD VM-exits for shadow VMCS fields and
9066	* thus might not need to import the shadow VMCS state, it's safer just in case
9067	* code elsewhere dares look at unsynced VMCS fields.
9068	*/
9069	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9070	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
9071	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
9072	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
9073	\| CPUMCTX_EXTRN_HWVIRT
9074	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
9075	AssertRCReturn(rc, rc);
9076
9077	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
9078
9079	VMXVEXITINFO ExitInfo;
9080	RT_ZERO(ExitInfo);
9081	ExitInfo.uReason = pVmxTransient->uExitReason;
9082	ExitInfo.u64Qual = pVmxTransient->uExitQual;
9083	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
9084	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
9085	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
9086	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
9087
9088	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
9089	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9090	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
9091	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9092	{
9093	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9094	rcStrict = VINF_SUCCESS;
9095	}
9096	return rcStrict;
9097	}
9098
9099
9100	/**
9101	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
9102	*/
9103	HMVMX_EXIT_DECL vmxHCExitVmresume(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9104	{
9105	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9106
9107	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
9108	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
9109	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9110	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
9111	AssertRCReturn(rc, rc);
9112
9113	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
9114
9115	STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitVmentry, z);
9116	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbExitInstr, VMXINSTRID_VMRESUME);
9117	STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitVmentry, z);
9118	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9119	{
9120	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
9121	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
9122	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
9123	}
9124	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
9125	return rcStrict;
9126	}
9127
9128
9129	/**
9130	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Conditional VM-exit.
9131	*/
9132	HMVMX_EXIT_DECL vmxHCExitVmwrite(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9133	{
9134	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9135
9136	/*
9137	* Although we should not get VMWRITE VM-exits for shadow VMCS fields, since our HM hook
9138	* gets invoked when IEM's VMWRITE instruction emulation modifies the current VMCS and it
9139	* flags re-loading the entire shadow VMCS, we should save the entire shadow VMCS here.
9140	*/
9141	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9142	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
9143	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
9144	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
9145	\| CPUMCTX_EXTRN_HWVIRT
9146	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
9147	AssertRCReturn(rc, rc);
9148
9149	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
9150
9151	VMXVEXITINFO ExitInfo;
9152	RT_ZERO(ExitInfo);
9153	ExitInfo.uReason = pVmxTransient->uExitReason;
9154	ExitInfo.u64Qual = pVmxTransient->uExitQual;
9155	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
9156	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
9157	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
9158	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
9159
9160	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
9161	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9162	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
9163	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9164	{
9165	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9166	rcStrict = VINF_SUCCESS;
9167	}
9168	return rcStrict;
9169	}
9170
9171
9172	/**
9173	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
9174	*/
9175	HMVMX_EXIT_DECL vmxHCExitVmxoff(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9176	{
9177	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9178
9179	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9180	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
9181	\| CPUMCTX_EXTRN_HWVIRT
9182	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
9183	AssertRCReturn(rc, rc);
9184
9185	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
9186
9187	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbExitInstr);
9188	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9189	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
9190	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9191	{
9192	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9193	rcStrict = VINF_SUCCESS;
9194	}
9195	return rcStrict;
9196	}
9197
9198
9199	/**
9200	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
9201	*/
9202	HMVMX_EXIT_DECL vmxHCExitVmxon(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9203	{
9204	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9205
9206	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9207	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
9208	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
9209	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
9210	\| CPUMCTX_EXTRN_HWVIRT
9211	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
9212	AssertRCReturn(rc, rc);
9213
9214	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
9215
9216	VMXVEXITINFO ExitInfo;
9217	RT_ZERO(ExitInfo);
9218	ExitInfo.uReason = pVmxTransient->uExitReason;
9219	ExitInfo.u64Qual = pVmxTransient->uExitQual;
9220	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
9221	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
9222	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
9223
9224	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
9225	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9226	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
9227	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9228	{
9229	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9230	rcStrict = VINF_SUCCESS;
9231	}
9232	return rcStrict;
9233	}
9234
9235
9236	/**
9237	* VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
9238	*/
9239	HMVMX_EXIT_DECL vmxHCExitInvvpid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9240	{
9241	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9242
9243	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9244	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
9245	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
9246	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
9247	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
9248	AssertRCReturn(rc, rc);
9249
9250	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
9251
9252	VMXVEXITINFO ExitInfo;
9253	RT_ZERO(ExitInfo);
9254	ExitInfo.uReason = pVmxTransient->uExitReason;
9255	ExitInfo.u64Qual = pVmxTransient->uExitQual;
9256	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
9257	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
9258	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
9259
9260	VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
9261	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9262	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
9263	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9264	{
9265	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9266	rcStrict = VINF_SUCCESS;
9267	}
9268	return rcStrict;
9269	}
9270
9271
9272	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
9273	/**
9274	* VM-exit handler for INVEPT (VMX_EXIT_INVEPT). Unconditional VM-exit.
9275	*/
9276	HMVMX_EXIT_DECL vmxHCExitInvept(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9277	{
9278	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9279
9280	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9281	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
9282	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
9283	int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
9284	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
9285	AssertRCReturn(rc, rc);
9286
9287	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
9288
9289	VMXVEXITINFO ExitInfo;
9290	RT_ZERO(ExitInfo);
9291	ExitInfo.uReason = pVmxTransient->uExitReason;
9292	ExitInfo.u64Qual = pVmxTransient->uExitQual;
9293	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
9294	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
9295	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
9296
9297	VBOXSTRICTRC rcStrict = IEMExecDecodedInvept(pVCpu, &ExitInfo);
9298	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9299	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
9300	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9301	{
9302	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9303	rcStrict = VINF_SUCCESS;
9304	}
9305	return rcStrict;
9306	}
9307	# endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
9308	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
9309	/** @} */
9310
9311
9312	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9313	/** @name Nested-guest VM-exit handlers.
9314	* @{
9315	*/
9316	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9317	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- Nested-guest VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9318	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9319
9320	/**
9321	* Nested-guest VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9322	* Conditional VM-exit.
9323	*/
9324	HMVMX_EXIT_DECL vmxHCExitXcptOrNmiNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9325	{
9326	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9327
9328	vmxHCReadExitIntInfoVmcs(pVCpu, pVmxTransient);
9329
9330	uint64_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
9331	uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
9332	Assert(VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo));
9333
9334	switch (uExitIntType)
9335	{
9336	#ifndef IN_NEM_DARWIN
9337	/*
9338	* Physical NMIs:
9339	* We shouldn't direct host physical NMIs to the nested-guest. Dispatch it to the host.
9340	*/
9341	case VMX_EXIT_INT_INFO_TYPE_NMI:
9342	return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
9343	#endif
9344
9345	/*
9346	* Hardware exceptions,
9347	* Software exceptions,
9348	* Privileged software exceptions:
9349	* Figure out if the exception must be delivered to the guest or the nested-guest.
9350	*/
9351	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
9352	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
9353	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
9354	{
9355	vmxHCReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
9356	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9357	vmxHCReadIdtVectoringInfoVmcs(pVCpu, pVmxTransient);
9358	vmxHCReadIdtVectoringErrorCodeVmcs(pVCpu, pVmxTransient);
9359
9360	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9361	bool const fIntercept = CPUMIsGuestVmxXcptInterceptSet(pCtx, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo),
9362	pVmxTransient->uExitIntErrorCode);
9363	if (fIntercept)
9364	{
9365	/* Exit qualification is required for debug and page-fault exceptions. */
9366	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
9367
9368	/*
9369	* For VM-exits due to software exceptions (those generated by INT3 or INTO) and privileged
9370	* software exceptions (those generated by INT1/ICEBP) we need to supply the VM-exit instruction
9371	* length. However, if delivery of a software interrupt, software exception or privileged
9372	* software exception causes a VM-exit, that too provides the VM-exit instruction length.
9373	*/
9374	VMXVEXITINFO ExitInfo;
9375	RT_ZERO(ExitInfo);
9376	ExitInfo.uReason = pVmxTransient->uExitReason;
9377	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
9378	ExitInfo.u64Qual = pVmxTransient->uExitQual;
9379
9380	VMXVEXITEVENTINFO ExitEventInfo;
9381	RT_ZERO(ExitEventInfo);
9382	ExitEventInfo.uExitIntInfo = pVmxTransient->uExitIntInfo;
9383	ExitEventInfo.uExitIntErrCode = pVmxTransient->uExitIntErrorCode;
9384	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
9385	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
9386
9387	#ifdef DEBUG_ramshankar
9388	vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
9389	Log4Func(("exit_int_info=%#RX32 err_code=%#RX32 exit_qual=%#RX64\n", pVmxTransient->uExitIntInfo,
9390	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual));
9391	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
9392	{
9393	Log4Func(("idt_info=%#RX32 idt_errcode=%#RX32 cr2=%#RX64\n", pVmxTransient->uIdtVectoringInfo,
9394	pVmxTransient->uIdtVectoringErrorCode, pCtx->cr2));
9395	}
9396	#endif
9397	return IEMExecVmxVmexitXcpt(pVCpu, &ExitInfo, &ExitEventInfo);
9398	}
9399
9400	/* Nested paging is currently a requirement, otherwise we would need to handle shadow #PFs in vmxHCExitXcptPF. */
9401	Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
9402	return vmxHCExitXcpt(pVCpu, pVmxTransient);
9403	}
9404
9405	/*
9406	* Software interrupts:
9407	* VM-exits cannot be caused by software interrupts.
9408	*
9409	* External interrupts:
9410	* This should only happen when "acknowledge external interrupts on VM-exit"
9411	* control is set. However, we never set this when executing a guest or
9412	* nested-guest. For nested-guests it is emulated while injecting interrupts into
9413	* the guest.
9414	*/
9415	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
9416	case VMX_EXIT_INT_INFO_TYPE_EXT_INT:
9417	default:
9418	{
9419	VCPU_2_VMXSTATE(pVCpu).u32HMError = pVmxTransient->uExitIntInfo;
9420	return VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9421	}
9422	}
9423	}
9424
9425
9426	/**
9427	* Nested-guest VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT).
9428	* Unconditional VM-exit.
9429	*/
9430	HMVMX_EXIT_DECL vmxHCExitTripleFaultNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9431	{
9432	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9433	return IEMExecVmxVmexitTripleFault(pVCpu);
9434	}
9435
9436
9437	/**
9438	* Nested-guest VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9439	*/
9440	HMVMX_EXIT_NSRC_DECL vmxHCExitIntWindowNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9441	{
9442	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9443
9444	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INT_WINDOW_EXIT))
9445	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
9446	return vmxHCExitIntWindow(pVCpu, pVmxTransient);
9447	}
9448
9449
9450	/**
9451	* Nested-guest VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9452	*/
9453	HMVMX_EXIT_NSRC_DECL vmxHCExitNmiWindowNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9454	{
9455	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9456
9457	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_NMI_WINDOW_EXIT))
9458	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
9459	return vmxHCExitIntWindow(pVCpu, pVmxTransient);
9460	}
9461
9462
9463	/**
9464	* Nested-guest VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH).
9465	* Unconditional VM-exit.
9466	*/
9467	HMVMX_EXIT_DECL vmxHCExitTaskSwitchNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9468	{
9469	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9470
9471	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
9472	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9473	vmxHCReadIdtVectoringInfoVmcs(pVCpu, pVmxTransient);
9474	vmxHCReadIdtVectoringErrorCodeVmcs(pVCpu, pVmxTransient);
9475
9476	VMXVEXITINFO ExitInfo;
9477	RT_ZERO(ExitInfo);
9478	ExitInfo.uReason = pVmxTransient->uExitReason;
9479	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
9480	ExitInfo.u64Qual = pVmxTransient->uExitQual;
9481
9482	VMXVEXITEVENTINFO ExitEventInfo;
9483	RT_ZERO(ExitEventInfo);
9484	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
9485	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
9486	return IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
9487	}
9488
9489
9490	/**
9491	* Nested-guest VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9492	*/
9493	HMVMX_EXIT_DECL vmxHCExitHltNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9494	{
9495	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9496
9497	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
9498	{
9499	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9500	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
9501	}
9502	return vmxHCExitHlt(pVCpu, pVmxTransient);
9503	}
9504
9505
9506	/**
9507	* Nested-guest VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9508	*/
9509	HMVMX_EXIT_DECL vmxHCExitInvlpgNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9510	{
9511	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9512
9513	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
9514	{
9515	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9516	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
9517
9518	VMXVEXITINFO ExitInfo;
9519	RT_ZERO(ExitInfo);
9520	ExitInfo.uReason = pVmxTransient->uExitReason;
9521	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
9522	ExitInfo.u64Qual = pVmxTransient->uExitQual;
9523	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
9524	}
9525	return vmxHCExitInvlpg(pVCpu, pVmxTransient);
9526	}
9527
9528
9529	/**
9530	* Nested-guest VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9531	*/
9532	HMVMX_EXIT_DECL vmxHCExitRdpmcNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9533	{
9534	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9535
9536	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDPMC_EXIT))
9537	{
9538	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9539	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
9540	}
9541	return vmxHCExitRdpmc(pVCpu, pVmxTransient);
9542	}
9543
9544
9545	/**
9546	* Nested-guest VM-exit handler for VMREAD (VMX_EXIT_VMREAD) and VMWRITE
9547	* (VMX_EXIT_VMWRITE). Conditional VM-exit.
9548	*/
9549	HMVMX_EXIT_DECL vmxHCExitVmreadVmwriteNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9550	{
9551	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9552
9553	Assert( pVmxTransient->uExitReason == VMX_EXIT_VMREAD
9554	\|\| pVmxTransient->uExitReason == VMX_EXIT_VMWRITE);
9555
9556	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
9557
9558	uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
9559	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
9560	uint64_t u64VmcsField = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
9561
9562	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
9563	if (!CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
9564	u64VmcsField &= UINT64_C(0xffffffff);
9565
9566	if (CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, pVmxTransient->uExitReason, u64VmcsField))
9567	{
9568	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9569	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
9570
9571	VMXVEXITINFO ExitInfo;
9572	RT_ZERO(ExitInfo);
9573	ExitInfo.uReason = pVmxTransient->uExitReason;
9574	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
9575	ExitInfo.u64Qual = pVmxTransient->uExitQual;
9576	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
9577	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
9578	}
9579
9580	if (pVmxTransient->uExitReason == VMX_EXIT_VMREAD)
9581	return vmxHCExitVmread(pVCpu, pVmxTransient);
9582	return vmxHCExitVmwrite(pVCpu, pVmxTransient);
9583	}
9584
9585
9586	/**
9587	* Nested-guest VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
9588	*/
9589	HMVMX_EXIT_DECL vmxHCExitRdtscNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9590	{
9591	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9592
9593	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
9594	{
9595	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9596	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
9597	}
9598
9599	return vmxHCExitRdtsc(pVCpu, pVmxTransient);
9600	}
9601
9602
9603	/**
9604	* Nested-guest VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX).
9605	* Conditional VM-exit.
9606	*/
9607	HMVMX_EXIT_DECL vmxHCExitMovCRxNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9608	{
9609	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9610
9611	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
9612	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9613
9614	VBOXSTRICTRC rcStrict;
9615	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual);
9616	switch (uAccessType)
9617	{
9618	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
9619	{
9620	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
9621	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
9622	Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
9623	uint64_t const uNewCrX = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
9624
9625	bool fIntercept;
9626	switch (iCrReg)
9627	{
9628	case 0:
9629	case 4:
9630	fIntercept = CPUMIsGuestVmxMovToCr0Cr4InterceptSet(&pVCpu->cpum.GstCtx, iCrReg, uNewCrX);
9631	break;
9632
9633	case 3:
9634	fIntercept = CPUMIsGuestVmxMovToCr3InterceptSet(pVCpu, uNewCrX);
9635	break;
9636
9637	case 8:
9638	fIntercept = CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_CR8_LOAD_EXIT);
9639	break;
9640
9641	default:
9642	fIntercept = false;
9643	break;
9644	}
9645	if (fIntercept)
9646	{
9647	VMXVEXITINFO ExitInfo;
9648	RT_ZERO(ExitInfo);
9649	ExitInfo.uReason = pVmxTransient->uExitReason;
9650	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
9651	ExitInfo.u64Qual = pVmxTransient->uExitQual;
9652	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
9653	}
9654	else
9655	{
9656	int const rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
9657	AssertRCReturn(rc, rc);
9658	rcStrict = vmxHCExitMovToCrX(pVCpu, pVmxTransient->cbExitInstr, iGReg, iCrReg);
9659	}
9660	break;
9661	}
9662
9663	case VMX_EXIT_QUAL_CRX_ACCESS_READ:
9664	{
9665	/*
9666	* CR0/CR4 reads do not cause VM-exits, the read-shadow is used (subject to masking).
9667	* CR2 reads do not cause a VM-exit.
9668	* CR3 reads cause a VM-exit depending on the "CR3 store exiting" control.
9669	* CR8 reads cause a VM-exit depending on the "CR8 store exiting" control.
9670	*/
9671	uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
9672	if ( iCrReg == 3
9673	\|\| iCrReg == 8)
9674	{
9675	static const uint32_t s_auCrXReadIntercepts[] = { 0, 0, 0, VMX_PROC_CTLS_CR3_STORE_EXIT, 0,
9676	0, 0, 0, VMX_PROC_CTLS_CR8_STORE_EXIT };
9677	uint32_t const uIntercept = s_auCrXReadIntercepts[iCrReg];
9678	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, uIntercept))
9679	{
9680	VMXVEXITINFO ExitInfo;
9681	RT_ZERO(ExitInfo);
9682	ExitInfo.uReason = pVmxTransient->uExitReason;
9683	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
9684	ExitInfo.u64Qual = pVmxTransient->uExitQual;
9685	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
9686	}
9687	else
9688	{
9689	uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
9690	rcStrict = vmxHCExitMovFromCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
9691	}
9692	}
9693	else
9694	{
9695	AssertMsgFailed(("MOV from CR%d VM-exit must not happen\n", iCrReg));
9696	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, iCrReg);
9697	}
9698	break;
9699	}
9700
9701	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
9702	{
9703	PCVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
9704	uint64_t const uGstHostMask = pVmcsNstGst->u64Cr0Mask.u;
9705	uint64_t const uReadShadow = pVmcsNstGst->u64Cr0ReadShadow.u;
9706	if ( (uGstHostMask & X86_CR0_TS)
9707	&& (uReadShadow & X86_CR0_TS))
9708	{
9709	VMXVEXITINFO ExitInfo;
9710	RT_ZERO(ExitInfo);
9711	ExitInfo.uReason = pVmxTransient->uExitReason;
9712	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
9713	ExitInfo.u64Qual = pVmxTransient->uExitQual;
9714	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
9715	}
9716	else
9717	rcStrict = vmxHCExitClts(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr);
9718	break;
9719	}
9720
9721	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
9722	{
9723	RTGCPTR GCPtrEffDst;
9724	uint16_t const uNewMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(pVmxTransient->uExitQual);
9725	bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(pVmxTransient->uExitQual);
9726	if (fMemOperand)
9727	{
9728	vmxHCReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
9729	GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
9730	}
9731	else
9732	GCPtrEffDst = NIL_RTGCPTR;
9733
9734	if (CPUMIsGuestVmxLmswInterceptSet(&pVCpu->cpum.GstCtx, uNewMsw))
9735	{
9736	VMXVEXITINFO ExitInfo;
9737	RT_ZERO(ExitInfo);
9738	ExitInfo.uReason = pVmxTransient->uExitReason;
9739	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
9740	ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
9741	ExitInfo.u64Qual = pVmxTransient->uExitQual;
9742	rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
9743	}
9744	else
9745	rcStrict = vmxHCExitLmsw(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, uNewMsw, GCPtrEffDst);
9746	break;
9747	}
9748
9749	default:
9750	{
9751	AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
9752	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
9753	}
9754	}
9755
9756	if (rcStrict == VINF_IEM_RAISED_XCPT)
9757	{
9758	ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9759	rcStrict = VINF_SUCCESS;
9760	}
9761	return rcStrict;
9762	}
9763
9764
9765	/**
9766	* Nested-guest VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX).
9767	* Conditional VM-exit.
9768	*/
9769	HMVMX_EXIT_DECL vmxHCExitMovDRxNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9770	{
9771	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9772
9773	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MOV_DR_EXIT))
9774	{
9775	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
9776	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9777
9778	VMXVEXITINFO ExitInfo;
9779	RT_ZERO(ExitInfo);
9780	ExitInfo.uReason = pVmxTransient->uExitReason;
9781	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
9782	ExitInfo.u64Qual = pVmxTransient->uExitQual;
9783	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
9784	}
9785	return vmxHCExitMovDRx(pVCpu, pVmxTransient);
9786	}
9787
9788
9789	/**
9790	* Nested-guest VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR).
9791	* Conditional VM-exit.
9792	*/
9793	HMVMX_EXIT_DECL vmxHCExitIoInstrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9794	{
9795	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9796
9797	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
9798
9799	uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
9800	uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
9801	AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
9802
9803	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */
9804	uint8_t const cbAccess = s_aIOSizes[uIOSize];
9805	if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
9806	{
9807	/*
9808	* IN/OUT instruction:
9809	* - Provides VM-exit instruction length.
9810	*
9811	* INS/OUTS instruction:
9812	* - Provides VM-exit instruction length.
9813	* - Provides Guest-linear address.
9814	* - Optionally provides VM-exit instruction info (depends on CPU feature).
9815	*/
9816	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
9817	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9818
9819	/* Make sure we don't use stale/uninitialized VMX-transient info. below. */
9820	pVmxTransient->ExitInstrInfo.u = 0;
9821	pVmxTransient->uGuestLinearAddr = 0;
9822
9823	bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
9824	bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
9825	if (fIOString)
9826	{
9827	vmxHCReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
9828	if (fVmxInsOutsInfo)
9829	{
9830	Assert(RT_BF_GET(g_HmMsrs.u.vmx.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
9831	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
9832	}
9833	}
9834
9835	VMXVEXITINFO ExitInfo;
9836	RT_ZERO(ExitInfo);
9837	ExitInfo.uReason = pVmxTransient->uExitReason;
9838	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
9839	ExitInfo.u64Qual = pVmxTransient->uExitQual;
9840	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
9841	ExitInfo.u64GuestLinearAddr = pVmxTransient->uGuestLinearAddr;
9842	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
9843	}
9844	return vmxHCExitIoInstr(pVCpu, pVmxTransient);
9845	}
9846
9847
9848	/**
9849	* Nested-guest VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9850	*/
9851	HMVMX_EXIT_DECL vmxHCExitRdmsrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9852	{
9853	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9854
9855	uint32_t fMsrpm;
9856	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
9857	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.abMsrBitmap, pVCpu->cpum.GstCtx.ecx);
9858	else
9859	fMsrpm = VMXMSRPM_EXIT_RD;
9860
9861	if (fMsrpm & VMXMSRPM_EXIT_RD)
9862	{
9863	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9864	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
9865	}
9866	return vmxHCExitRdmsr(pVCpu, pVmxTransient);
9867	}
9868
9869
9870	/**
9871	* Nested-guest VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
9872	*/
9873	HMVMX_EXIT_DECL vmxHCExitWrmsrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9874	{
9875	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9876
9877	uint32_t fMsrpm;
9878	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
9879	fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.abMsrBitmap, pVCpu->cpum.GstCtx.ecx);
9880	else
9881	fMsrpm = VMXMSRPM_EXIT_WR;
9882
9883	if (fMsrpm & VMXMSRPM_EXIT_WR)
9884	{
9885	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9886	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
9887	}
9888	return vmxHCExitWrmsr(pVCpu, pVmxTransient);
9889	}
9890
9891
9892	/**
9893	* Nested-guest VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9894	*/
9895	HMVMX_EXIT_DECL vmxHCExitMwaitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9896	{
9897	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9898
9899	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MWAIT_EXIT))
9900	{
9901	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9902	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
9903	}
9904	return vmxHCExitMwait(pVCpu, pVmxTransient);
9905	}
9906
9907
9908	/**
9909	* Nested-guest VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional
9910	* VM-exit.
9911	*/
9912	HMVMX_EXIT_DECL vmxHCExitMtfNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9913	{
9914	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9915
9916	/** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */
9917	vmxHCReadGuestPendingDbgXctps(pVCpu, pVmxTransient);
9918	VMXVEXITINFO ExitInfo;
9919	RT_ZERO(ExitInfo);
9920	ExitInfo.uReason = pVmxTransient->uExitReason;
9921	ExitInfo.u64GuestPendingDbgXcpts = pVmxTransient->uGuestPendingDbgXcpts;
9922	return IEMExecVmxVmexitTrapLike(pVCpu, &ExitInfo);
9923	}
9924
9925
9926	/**
9927	* Nested-guest VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9928	*/
9929	HMVMX_EXIT_DECL vmxHCExitMonitorNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9930	{
9931	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9932
9933	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MONITOR_EXIT))
9934	{
9935	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9936	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
9937	}
9938	return vmxHCExitMonitor(pVCpu, pVmxTransient);
9939	}
9940
9941
9942	/**
9943	* Nested-guest VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
9944	*/
9945	HMVMX_EXIT_DECL vmxHCExitPauseNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9946	{
9947	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9948
9949	/** @todo NSTVMX: Think about this more. Does the outer guest need to intercept
9950	* PAUSE when executing a nested-guest? If it does not, we would not need
9951	* to check for the intercepts here. Just call VM-exit... */
9952
9953	/* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */
9954	if ( CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT)
9955	\|\| CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
9956	{
9957	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9958	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
9959	}
9960	return vmxHCExitPause(pVCpu, pVmxTransient);
9961	}
9962
9963
9964	/**
9965	* Nested-guest VM-exit handler for when the TPR value is lowered below the
9966	* specified threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
9967	*/
9968	HMVMX_EXIT_NSRC_DECL vmxHCExitTprBelowThresholdNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9969	{
9970	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9971
9972	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_TPR_SHADOW))
9973	{
9974	vmxHCReadGuestPendingDbgXctps(pVCpu, pVmxTransient);
9975	VMXVEXITINFO ExitInfo;
9976	RT_ZERO(ExitInfo);
9977	ExitInfo.uReason = pVmxTransient->uExitReason;
9978	ExitInfo.u64GuestPendingDbgXcpts = pVmxTransient->uGuestPendingDbgXcpts;
9979	return IEMExecVmxVmexitTrapLike(pVCpu, &ExitInfo);
9980	}
9981	return vmxHCExitTprBelowThreshold(pVCpu, pVmxTransient);
9982	}
9983
9984
9985	/**
9986	* Nested-guest VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional
9987	* VM-exit.
9988	*/
9989	HMVMX_EXIT_DECL vmxHCExitApicAccessNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
9990	{
9991	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
9992
9993	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9994	vmxHCReadIdtVectoringInfoVmcs(pVCpu, pVmxTransient);
9995	vmxHCReadIdtVectoringErrorCodeVmcs(pVCpu, pVmxTransient);
9996	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
9997
9998	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS));
9999
10000	Log4Func(("at offset %#x type=%u\n", VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual),
10001	VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual)));
10002
10003	VMXVEXITINFO ExitInfo;
10004	RT_ZERO(ExitInfo);
10005	ExitInfo.uReason = pVmxTransient->uExitReason;
10006	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
10007	ExitInfo.u64Qual = pVmxTransient->uExitQual;
10008
10009	VMXVEXITEVENTINFO ExitEventInfo;
10010	RT_ZERO(ExitEventInfo);
10011	ExitEventInfo.uIdtVectoringInfo = pVmxTransient->uIdtVectoringInfo;
10012	ExitEventInfo.uIdtVectoringErrCode = pVmxTransient->uIdtVectoringErrorCode;
10013	return IEMExecVmxVmexitApicAccess(pVCpu, &ExitInfo, &ExitEventInfo);
10014	}
10015
10016
10017	/**
10018	* Nested-guest VM-exit handler for APIC write emulation (VMX_EXIT_APIC_WRITE).
10019	* Conditional VM-exit.
10020	*/
10021	HMVMX_EXIT_DECL vmxHCExitApicWriteNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10022	{
10023	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10024
10025	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_APIC_REG_VIRT));
10026	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
10027	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
10028	}
10029
10030
10031	/**
10032	* Nested-guest VM-exit handler for virtualized EOI (VMX_EXIT_VIRTUALIZED_EOI).
10033	* Conditional VM-exit.
10034	*/
10035	HMVMX_EXIT_DECL vmxHCExitVirtEoiNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10036	{
10037	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10038
10039	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_INT_DELIVERY));
10040	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
10041	return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
10042	}
10043
10044
10045	/**
10046	* Nested-guest VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
10047	*/
10048	HMVMX_EXIT_DECL vmxHCExitRdtscpNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10049	{
10050	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10051
10052	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
10053	{
10054	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_RDTSCP));
10055	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10056	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
10057	}
10058	return vmxHCExitRdtscp(pVCpu, pVmxTransient);
10059	}
10060
10061
10062	/**
10063	* Nested-guest VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
10064	*/
10065	HMVMX_EXIT_NSRC_DECL vmxHCExitWbinvdNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10066	{
10067	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10068
10069	if (CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
10070	{
10071	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10072	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
10073	}
10074	return vmxHCExitWbinvd(pVCpu, pVmxTransient);
10075	}
10076
10077
10078	/**
10079	* Nested-guest VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
10080	*/
10081	HMVMX_EXIT_DECL vmxHCExitInvpcidNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10082	{
10083	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10084
10085	if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
10086	{
10087	Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_INVPCID));
10088	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10089	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
10090	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
10091
10092	VMXVEXITINFO ExitInfo;
10093	RT_ZERO(ExitInfo);
10094	ExitInfo.uReason = pVmxTransient->uExitReason;
10095	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
10096	ExitInfo.u64Qual = pVmxTransient->uExitQual;
10097	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
10098	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
10099	}
10100	return vmxHCExitInvpcid(pVCpu, pVmxTransient);
10101	}
10102
10103
10104	/**
10105	* Nested-guest VM-exit handler for invalid-guest state
10106	* (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error VM-exit.
10107	*/
10108	HMVMX_EXIT_DECL vmxHCExitErrInvalidGuestStateNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10109	{
10110	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10111
10112	/*
10113	* Currently this should never happen because we fully emulate VMLAUNCH/VMRESUME in IEM.
10114	* So if it does happen, it indicates a bug possibly in the hardware-assisted VMX code.
10115	* Handle it like it's in an invalid guest state of the outer guest.
10116	*
10117	* When the fast path is implemented, this should be changed to cause the corresponding
10118	* nested-guest VM-exit.
10119	*/
10120	return vmxHCExitErrInvalidGuestState(pVCpu, pVmxTransient);
10121	}
10122
10123
10124	/**
10125	* Nested-guest VM-exit handler for instructions that cause VM-exits uncondtionally
10126	* and only provide the instruction length.
10127	*
10128	* Unconditional VM-exit.
10129	*/
10130	HMVMX_EXIT_DECL vmxHCExitInstrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10131	{
10132	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10133
10134	#ifdef VBOX_STRICT
10135	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10136	switch (pVmxTransient->uExitReason)
10137	{
10138	case VMX_EXIT_ENCLS:
10139	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_ENCLS_EXIT));
10140	break;
10141
10142	case VMX_EXIT_VMFUNC:
10143	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_VMFUNC));
10144	break;
10145	}
10146	#endif
10147
10148	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10149	return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
10150	}
10151
10152
10153	/**
10154	* Nested-guest VM-exit handler for instructions that provide instruction length as
10155	* well as more information.
10156	*
10157	* Unconditional VM-exit.
10158	*/
10159	HMVMX_EXIT_DECL vmxHCExitInstrWithInfoNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
10160	{
10161	HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10162
10163	#ifdef VBOX_STRICT
10164	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10165	switch (pVmxTransient->uExitReason)
10166	{
10167	case VMX_EXIT_GDTR_IDTR_ACCESS:
10168	case VMX_EXIT_LDTR_TR_ACCESS:
10169	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_DESC_TABLE_EXIT));
10170	break;
10171
10172	case VMX_EXIT_RDRAND:
10173	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_RDRAND_EXIT));
10174	break;
10175
10176	case VMX_EXIT_RDSEED:
10177	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_RDSEED_EXIT));
10178	break;
10179
10180	case VMX_EXIT_XSAVES:
10181	case VMX_EXIT_XRSTORS:
10182	/** @todo NSTVMX: Verify XSS-bitmap. */
10183	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_XSAVES_XRSTORS));
10184	break;
10185
10186	case VMX_EXIT_UMWAIT:
10187	case VMX_EXIT_TPAUSE:
10188	Assert(CPUMIsGuestVmxProcCtlsSet(pCtx, VMX_PROC_CTLS_RDTSC_EXIT));
10189	Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_USER_WAIT_PAUSE));
10190	break;
10191
10192	case VMX_EXIT_LOADIWKEY:
10193	Assert(CPUMIsGuestVmxProcCtls3Set(pCtx, VMX_PROC_CTLS3_LOADIWKEY_EXIT));
10194	break;
10195	}
10196	#endif
10197
10198	vmxHCReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10199	vmxHCReadExitQualVmcs(pVCpu, pVmxTransient);
10200	vmxHCReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
10201
10202	VMXVEXITINFO ExitInfo;
10203	RT_ZERO(ExitInfo);
10204	ExitInfo.uReason = pVmxTransient->uExitReason;
10205	ExitInfo.cbInstr = pVmxTransient->cbExitInstr;
10206	ExitInfo.u64Qual = pVmxTransient->uExitQual;
10207	ExitInfo.InstrInfo = pVmxTransient->ExitInstrInfo;
10208	return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
10209	}
10210
10211	/** @} */
10212	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10213

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/VMXAllTemplate.cpp.h@ 93582

Download in other formats: