HMVMXR0.cpp@ 47565

Last change on this file since 47565 was 47512, checked in by vboxsync, 11 years ago
VMM: make the StatExitHostNmi release
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 393.4 KB

Line
1	/* $Id: HMVMXR0.cpp 47512 2013-08-01 16:30:19Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HMVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iem.h>
32	#include <VBox/vmm/iom.h>
33	#include <VBox/vmm/selm.h>
34	#include <VBox/vmm/tm.h>
35	#ifdef VBOX_WITH_REM
36	# include <VBox/vmm/rem.h>
37	#endif
38	#ifdef DEBUG_ramshankar
39	#define HMVMX_SAVE_FULL_GUEST_STATE
40	#define HMVMX_SYNC_FULL_GUEST_STATE
41	#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
42	#define HMVMX_ALWAYS_TRAP_PF
43	#endif
44
45
46	/*******************************************************************************
47	* Defined Constants And Macros *
48	*******************************************************************************/
49	#if defined(RT_ARCH_AMD64)
50	# define HMVMX_IS_64BIT_HOST_MODE() (true)
51	typedef RTHCUINTREG HMVMXHCUINTREG;
52	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
53	extern "C" uint32_t g_fVMXIs64bitHost;
54	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
55	typedef uint64_t HMVMXHCUINTREG;
56	#else
57	# define HMVMX_IS_64BIT_HOST_MODE() (false)
58	typedef RTHCUINTREG HMVMXHCUINTREG;
59	#endif
60
61	/** Use the function table. */
62	#define HMVMX_USE_FUNCTION_TABLE
63
64	/** Determine which tagged-TLB flush handler to use. */
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70	/** @name Updated-guest-state flags.
71	* @{ */
72	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
73	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
74	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
75	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
76	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
77	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
78	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
79	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
80	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
81	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
82	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
83	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
84	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
85	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
86	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
87	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
89	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
90	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
91	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
92	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
93	\| HMVMX_UPDATED_GUEST_RSP \
94	\| HMVMX_UPDATED_GUEST_RFLAGS \
95	\| HMVMX_UPDATED_GUEST_CR0 \
96	\| HMVMX_UPDATED_GUEST_CR3 \
97	\| HMVMX_UPDATED_GUEST_CR4 \
98	\| HMVMX_UPDATED_GUEST_GDTR \
99	\| HMVMX_UPDATED_GUEST_IDTR \
100	\| HMVMX_UPDATED_GUEST_LDTR \
101	\| HMVMX_UPDATED_GUEST_TR \
102	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
103	\| HMVMX_UPDATED_GUEST_DEBUG \
104	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
105	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
106	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
107	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
109	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
110	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
111	\| HMVMX_UPDATED_GUEST_APIC_STATE)
112	/** @} */
113
114	/** @name
115	* Flags to skip redundant reads of some common VMCS fields that are not part of
116	* the guest-CPU state but are in the transient structure.
117	*/
118	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
119	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
120	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
124	/** @} */
125
126	/**
127	* Exception bitmap mask for real-mode guests (real-on-v86).
128	*
129	* We need to intercept all exceptions manually (except #PF). #NM is also
130	* handled separately, see hmR0VmxLoadGuestControlRegs(). #PF need not be
131	* intercepted even in real-mode if we have Nested Paging support.
132	*/
133	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
134	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
135	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
136	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
137	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
138	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
139	\| RT_BIT(X86_XCPT_XF))
140
141	/**
142	* Exception bitmap mask for all contributory exceptions.
143	*
144	* Page fault is deliberately excluded here as it's conditional as to whether
145	* it's contributory or benign. Page faults are handled separately.
146	*/
147	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
148	\| RT_BIT(X86_XCPT_DE))
149
150	/** Maximum VM-instruction error number. */
151	#define HMVMX_INSTR_ERROR_MAX 28
152
153	/** Profiling macro. */
154	#ifdef HM_PROFILE_EXIT_DISPATCH
155	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
156	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
157	#else
158	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
159	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
160	#endif
161
162
163	/*******************************************************************************
164	* Structures and Typedefs *
165	*******************************************************************************/
166	/**
167	* VMX transient state.
168	*
169	* A state structure for holding miscellaneous information across
170	* VMX non-root operation and restored after the transition.
171	*/
172	typedef struct VMXTRANSIENT
173	{
174	/** The host's rflags/eflags. */
175	RTCCUINTREG uEFlags;
176	#if HC_ARCH_BITS == 32
177	uint32_t u32Alignment0;
178	#endif
179	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
180	uint64_t u64LStarMsr;
181	/** The guest's TPR value used for TPR shadowing. */
182	uint8_t u8GuestTpr;
183	/** Alignment. */
184	uint8_t abAlignment0[7];
185
186	/** The basic VM-exit reason. */
187	uint16_t uExitReason;
188	/** Alignment. */
189	uint16_t u16Alignment0;
190	/** The VM-exit interruption error code. */
191	uint32_t uExitIntrErrorCode;
192	/** The VM-exit exit qualification. */
193	uint64_t uExitQualification;
194
195	/** The VM-exit interruption-information field. */
196	uint32_t uExitIntrInfo;
197	/** The VM-exit instruction-length field. */
198	uint32_t cbInstr;
199	/** The VM-exit instruction-information field. */
200	union
201	{
202	/** Plain unsigned int representation. */
203	uint32_t u;
204	/** INS and OUTS information. */
205	struct
206	{
207	uint32_t u6Reserved0 : 6;
208	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
209	uint32_t u3AddrSize : 3;
210	uint32_t u5Reserved1 : 5;
211	/** The segment register (X86_SREG_XXX). */
212	uint32_t iSegReg : 3;
213	uint32_t uReserved2 : 14;
214	} StrIo;
215	} ExitInstrInfo;
216	/** Whether the VM-entry failed or not. */
217	bool fVMEntryFailed;
218	/** Alignment. */
219	uint8_t abAlignment1[3];
220
221	/** The VM-entry interruption-information field. */
222	uint32_t uEntryIntrInfo;
223	/** The VM-entry exception error code field. */
224	uint32_t uEntryXcptErrorCode;
225	/** The VM-entry instruction length field. */
226	uint32_t cbEntryInstr;
227
228	/** IDT-vectoring information field. */
229	uint32_t uIdtVectoringInfo;
230	/** IDT-vectoring error code. */
231	uint32_t uIdtVectoringErrorCode;
232
233	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
234	uint32_t fVmcsFieldsRead;
235	/** Whether TSC-offsetting should be setup before VM-entry. */
236	bool fUpdateTscOffsettingAndPreemptTimer;
237	/** Whether the VM-exit was caused by a page-fault during delivery of a
238	* contributory exception or a page-fault. */
239	bool fVectoringPF;
240	} VMXTRANSIENT;
241	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
242	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
243	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
244	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
245	/** Pointer to VMX transient state. */
246	typedef VMXTRANSIENT *PVMXTRANSIENT;
247
248
249	/**
250	* MSR-bitmap read permissions.
251	*/
252	typedef enum VMXMSREXITREAD
253	{
254	/** Reading this MSR causes a VM-exit. */
255	VMXMSREXIT_INTERCEPT_READ = 0xb,
256	/** Reading this MSR does not cause a VM-exit. */
257	VMXMSREXIT_PASSTHRU_READ
258	} VMXMSREXITREAD;
259
260	/**
261	* MSR-bitmap write permissions.
262	*/
263	typedef enum VMXMSREXITWRITE
264	{
265	/** Writing to this MSR causes a VM-exit. */
266	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
267	/** Writing to this MSR does not cause a VM-exit. */
268	VMXMSREXIT_PASSTHRU_WRITE
269	} VMXMSREXITWRITE;
270
271	/**
272	* VM-exit handler.
273	*
274	* @returns VBox status code.
275	* @param pVCpu Pointer to the VMCPU.
276	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
277	* out-of-sync. Make sure to update the required
278	* fields before using them.
279	* @param pVmxTransient Pointer to the VMX-transient structure.
280	*/
281	#ifndef HMVMX_USE_FUNCTION_TABLE
282	typedef int FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
283	#else
284	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
285	/** Pointer to VM-exit handler. */
286	typedef FNVMEXITHANDLER *PFNVMEXITHANDLER;
287	#endif
288
289
290	/*******************************************************************************
291	* Internal Functions *
292	*******************************************************************************/
293	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
294	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
295	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
296	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
297	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
298	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
299	#endif
300	#ifndef HMVMX_USE_FUNCTION_TABLE
301	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
302	# define HMVMX_EXIT_DECL static int
303	#else
304	# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
305	#endif
306
307	/** @name VM-exit handlers.
308	* @{
309	*/
310	static FNVMEXITHANDLER hmR0VmxExitXcptOrNmi;
311	static FNVMEXITHANDLER hmR0VmxExitExtInt;
312	static FNVMEXITHANDLER hmR0VmxExitTripleFault;
313	static FNVMEXITHANDLER hmR0VmxExitInitSignal;
314	static FNVMEXITHANDLER hmR0VmxExitSipi;
315	static FNVMEXITHANDLER hmR0VmxExitIoSmi;
316	static FNVMEXITHANDLER hmR0VmxExitSmi;
317	static FNVMEXITHANDLER hmR0VmxExitIntWindow;
318	static FNVMEXITHANDLER hmR0VmxExitNmiWindow;
319	static FNVMEXITHANDLER hmR0VmxExitTaskSwitch;
320	static FNVMEXITHANDLER hmR0VmxExitCpuid;
321	static FNVMEXITHANDLER hmR0VmxExitGetsec;
322	static FNVMEXITHANDLER hmR0VmxExitHlt;
323	static FNVMEXITHANDLER hmR0VmxExitInvd;
324	static FNVMEXITHANDLER hmR0VmxExitInvlpg;
325	static FNVMEXITHANDLER hmR0VmxExitRdpmc;
326	static FNVMEXITHANDLER hmR0VmxExitRdtsc;
327	static FNVMEXITHANDLER hmR0VmxExitRsm;
328	static FNVMEXITHANDLER hmR0VmxExitSetPendingXcptUD;
329	static FNVMEXITHANDLER hmR0VmxExitMovCRx;
330	static FNVMEXITHANDLER hmR0VmxExitMovDRx;
331	static FNVMEXITHANDLER hmR0VmxExitIoInstr;
332	static FNVMEXITHANDLER hmR0VmxExitRdmsr;
333	static FNVMEXITHANDLER hmR0VmxExitWrmsr;
334	static FNVMEXITHANDLER hmR0VmxExitErrInvalidGuestState;
335	static FNVMEXITHANDLER hmR0VmxExitErrMsrLoad;
336	static FNVMEXITHANDLER hmR0VmxExitErrUndefined;
337	static FNVMEXITHANDLER hmR0VmxExitMwait;
338	static FNVMEXITHANDLER hmR0VmxExitMtf;
339	static FNVMEXITHANDLER hmR0VmxExitMonitor;
340	static FNVMEXITHANDLER hmR0VmxExitPause;
341	static FNVMEXITHANDLER hmR0VmxExitErrMachineCheck;
342	static FNVMEXITHANDLER hmR0VmxExitTprBelowThreshold;
343	static FNVMEXITHANDLER hmR0VmxExitApicAccess;
344	static FNVMEXITHANDLER hmR0VmxExitXdtrAccess;
345	static FNVMEXITHANDLER hmR0VmxExitXdtrAccess;
346	static FNVMEXITHANDLER hmR0VmxExitEptViolation;
347	static FNVMEXITHANDLER hmR0VmxExitEptMisconfig;
348	static FNVMEXITHANDLER hmR0VmxExitRdtscp;
349	static FNVMEXITHANDLER hmR0VmxExitPreemptTimer;
350	static FNVMEXITHANDLER hmR0VmxExitWbinvd;
351	static FNVMEXITHANDLER hmR0VmxExitXsetbv;
352	static FNVMEXITHANDLER hmR0VmxExitRdrand;
353	static FNVMEXITHANDLER hmR0VmxExitInvpcid;
354	/** @} */
355
356	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
357	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
358	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
359	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
360	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
361	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
362	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
363
364
365	/*******************************************************************************
366	* Global Variables *
367	*******************************************************************************/
368	#ifdef HMVMX_USE_FUNCTION_TABLE
369
370	/**
371	* VMX_EXIT dispatch table.
372	*/
373	static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
374	{
375	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
376	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
377	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
378	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
379	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
380	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
381	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
382	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
383	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
384	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
385	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
386	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
387	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
388	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
389	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
390	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
391	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
392	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
393	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
394	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
395	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
396	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
397	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
398	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
399	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
400	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
401	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
402	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
403	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
404	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
405	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
406	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
407	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
408	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
409	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
410	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
411	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
412	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
413	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
414	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
415	/* 40 UNDEFINED */ hmR0VmxExitPause,
416	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
417	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
418	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
419	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
420	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
421	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
422	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
423	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
424	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
425	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
426	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
427	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
428	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
429	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
430	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
431	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
432	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
433	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
434	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
435	};
436	#endif /* HMVMX_USE_FUNCTION_TABLE */
437
438	#ifdef VBOX_STRICT
439	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
440	{
441	/* 0 */ "(Not Used)",
442	/* 1 */ "VMCALL executed in VMX root operation.",
443	/* 2 */ "VMCLEAR with invalid physical address.",
444	/* 3 */ "VMCLEAR with VMXON pointer.",
445	/* 4 */ "VMLAUNCH with non-clear VMCS.",
446	/* 5 */ "VMRESUME with non-launched VMCS.",
447	/* 6 */ "VMRESUME after VMXOFF",
448	/* 7 */ "VM entry with invalid control fields.",
449	/* 8 */ "VM entry with invalid host state fields.",
450	/* 9 */ "VMPTRLD with invalid physical address.",
451	/* 10 */ "VMPTRLD with VMXON pointer.",
452	/* 11 */ "VMPTRLD with incorrect revision identifier.",
453	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
454	/* 13 */ "VMWRITE to read-only VMCS component.",
455	/* 14 */ "(Not Used)",
456	/* 15 */ "VMXON executed in VMX root operation.",
457	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
458	/* 17 */ "VM entry with non-launched executing VMCS.",
459	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
460	/* 19 */ "VMCALL with non-clear VMCS.",
461	/* 20 */ "VMCALL with invalid VM-exit control fields.",
462	/* 21 */ "(Not Used)",
463	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
464	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
465	/* 24 */ "VMCALL with invalid SMM-monitor features.",
466	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
467	/* 26 */ "VM entry with events blocked by MOV SS.",
468	/* 27 */ "(Not Used)",
469	/* 28 */ "Invalid operand to INVEPT/INVVPID."
470	};
471	#endif /* VBOX_STRICT */
472
473
474
475	/**
476	* Updates the VM's last error record. If there was a VMX instruction error,
477	* reads the error data from the VMCS and updates VCPU's last error record as
478	* well.
479	*
480	* @param pVM Pointer to the VM.
481	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
482	* VERR_VMX_UNABLE_TO_START_VM or
483	* VERR_VMX_INVALID_VMCS_FIELD).
484	* @param rc The error code.
485	*/
486	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
487	{
488	AssertPtr(pVM);
489	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
490	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
491	{
492	AssertPtrReturnVoid(pVCpu);
493	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
494	}
495	pVM->hm.s.lLastError = rc;
496	}
497
498
499	/**
500	* Reads the VM-entry interruption-information field from the VMCS into the VMX
501	* transient structure.
502	*
503	* @returns VBox status code.
504	* @param pVmxTransient Pointer to the VMX transient structure.
505	*
506	* @remarks No-long-jump zone!!!
507	*/
508	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
509	{
510	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
511	AssertRCReturn(rc, rc);
512	return VINF_SUCCESS;
513	}
514
515
516	/**
517	* Reads the VM-entry exception error code field from the VMCS into
518	* the VMX transient structure.
519	*
520	* @returns VBox status code.
521	* @param pVmxTransient Pointer to the VMX transient structure.
522	*
523	* @remarks No-long-jump zone!!!
524	*/
525	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
526	{
527	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
528	AssertRCReturn(rc, rc);
529	return VINF_SUCCESS;
530	}
531
532
533	/**
534	* Reads the VM-entry exception error code field from the VMCS into
535	* the VMX transient structure.
536	*
537	* @returns VBox status code.
538	* @param pVCpu Pointer to the VMCPU.
539	* @param pVmxTransient Pointer to the VMX transient structure.
540	*
541	* @remarks No-long-jump zone!!!
542	*/
543	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
544	{
545	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
546	AssertRCReturn(rc, rc);
547	return VINF_SUCCESS;
548	}
549
550
551	/**
552	* Reads the VM-exit interruption-information field from the VMCS into the VMX
553	* transient structure.
554	*
555	* @returns VBox status code.
556	* @param pVCpu Pointer to the VMCPU.
557	* @param pVmxTransient Pointer to the VMX transient structure.
558	*/
559	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
560	{
561	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
562	{
563	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
564	AssertRCReturn(rc, rc);
565	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
566	}
567	return VINF_SUCCESS;
568	}
569
570
571	/**
572	* Reads the VM-exit interruption error code from the VMCS into the VMX
573	* transient structure.
574	*
575	* @returns VBox status code.
576	* @param pVCpu Pointer to the VMCPU.
577	* @param pVmxTransient Pointer to the VMX transient structure.
578	*/
579	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
580	{
581	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
582	{
583	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
584	AssertRCReturn(rc, rc);
585	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
586	}
587	return VINF_SUCCESS;
588	}
589
590
591	/**
592	* Reads the VM-exit instruction length field from the VMCS into the VMX
593	* transient structure.
594	*
595	* @returns VBox status code.
596	* @param pVCpu Pointer to the VMCPU.
597	* @param pVmxTransient Pointer to the VMX transient structure.
598	*/
599	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
600	{
601	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
602	{
603	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
604	AssertRCReturn(rc, rc);
605	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
606	}
607	return VINF_SUCCESS;
608	}
609
610
611	/**
612	* Reads the VM-exit instruction-information field from the VMCS into
613	* the VMX transient structure.
614	*
615	* @returns VBox status code.
616	* @param pVCpu The cross context per CPU structure.
617	* @param pVmxTransient Pointer to the VMX transient structure.
618	*/
619	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
620	{
621	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
622	{
623	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->cbInstr);
624	AssertRCReturn(rc, rc);
625	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
626	}
627	return VINF_SUCCESS;
628	}
629
630
631	/**
632	* Reads the exit qualification from the VMCS into the VMX transient structure.
633	*
634	* @returns VBox status code.
635	* @param pVCpu Pointer to the VMCPU.
636	* @param pVmxTransient Pointer to the VMX transient structure.
637	*/
638	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
639	{
640	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
641	{
642	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
643	AssertRCReturn(rc, rc);
644	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
645	}
646	return VINF_SUCCESS;
647	}
648
649
650	/**
651	* Reads the IDT-vectoring information field from the VMCS into the VMX
652	* transient structure.
653	*
654	* @returns VBox status code.
655	* @param pVmxTransient Pointer to the VMX transient structure.
656	*
657	* @remarks No-long-jump zone!!!
658	*/
659	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
660	{
661	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
662	{
663	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
664	AssertRCReturn(rc, rc);
665	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
666	}
667	return VINF_SUCCESS;
668	}
669
670
671	/**
672	* Reads the IDT-vectoring error code from the VMCS into the VMX
673	* transient structure.
674	*
675	* @returns VBox status code.
676	* @param pVmxTransient Pointer to the VMX transient structure.
677	*/
678	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
679	{
680	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
681	{
682	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
683	AssertRCReturn(rc, rc);
684	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
685	}
686	return VINF_SUCCESS;
687	}
688
689
690	/**
691	* Enters VMX root mode operation on the current CPU.
692	*
693	* @returns VBox status code.
694	* @param pVM Pointer to the VM (optional, can be NULL, after
695	* a resume).
696	* @param HCPhysCpuPage Physical address of the VMXON region.
697	* @param pvCpuPage Pointer to the VMXON region.
698	*/
699	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
700	{
701	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
702	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
703	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
704
705	if (pVM)
706	{
707	/* Write the VMCS revision dword to the VMXON region. */
708	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
709	}
710
711	/* Enable the VMX bit in CR4 if necessary. */
712	RTCCUINTREG uCr4 = ASMGetCR4();
713	if (!(uCr4 & X86_CR4_VMXE))
714	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
715
716	/* Enter VMX root mode. */
717	int rc = VMXEnable(HCPhysCpuPage);
718	if (RT_FAILURE(rc))
719	ASMSetCR4(uCr4);
720
721	return rc;
722	}
723
724
725	/**
726	* Exits VMX root mode operation on the current CPU.
727	*
728	* @returns VBox status code.
729	*/
730	static int hmR0VmxLeaveRootMode(void)
731	{
732	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
733
734	/* If we're for some reason not in VMX root mode, then don't leave it. */
735	RTCCUINTREG uHostCR4 = ASMGetCR4();
736	if (uHostCR4 & X86_CR4_VMXE)
737	{
738	/* Exit VMX root mode and clear the VMX bit in CR4. */
739	VMXDisable();
740	ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
741	return VINF_SUCCESS;
742	}
743
744	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
745	}
746
747
748	/**
749	* Allocates and maps one physically contiguous page. The allocated page is
750	* zero'd out. (Used by various VT-x structures).
751	*
752	* @returns IPRT status code.
753	* @param pMemObj Pointer to the ring-0 memory object.
754	* @param ppVirt Where to store the virtual address of the
755	* allocation.
756	* @param pPhys Where to store the physical address of the
757	* allocation.
758	*/
759	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
760	{
761	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
762	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
763	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
764
765	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
766	if (RT_FAILURE(rc))
767	return rc;
768	ppVirt = RTR0MemObjAddress(pMemObj);
769	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
770	ASMMemZero32(*ppVirt, PAGE_SIZE);
771	return VINF_SUCCESS;
772	}
773
774
775	/**
776	* Frees and unmaps an allocated physical page.
777	*
778	* @param pMemObj Pointer to the ring-0 memory object.
779	* @param ppVirt Where to re-initialize the virtual address of
780	* allocation as 0.
781	* @param pHCPhys Where to re-initialize the physical address of the
782	* allocation as 0.
783	*/
784	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
785	{
786	AssertPtr(pMemObj);
787	AssertPtr(ppVirt);
788	AssertPtr(pHCPhys);
789	if (*pMemObj != NIL_RTR0MEMOBJ)
790	{
791	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
792	AssertRC(rc);
793	*pMemObj = NIL_RTR0MEMOBJ;
794	*ppVirt = 0;
795	*pHCPhys = 0;
796	}
797	}
798
799
800	/**
801	* Worker function to free VT-x related structures.
802	*
803	* @returns IPRT status code.
804	* @param pVM Pointer to the VM.
805	*/
806	static void hmR0VmxStructsFree(PVM pVM)
807	{
808	for (VMCPUID i = 0; i < pVM->cCpus; i++)
809	{
810	PVMCPU pVCpu = &pVM->aCpus[i];
811	AssertPtr(pVCpu);
812
813	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
814	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
815	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
816	#endif
817
818	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
819	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
820
821	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
822	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
823	}
824
825	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
826	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
827	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
828	#endif
829	}
830
831
832	/**
833	* Worker function to allocate VT-x related VM structures.
834	*
835	* @returns IPRT status code.
836	* @param pVM Pointer to the VM.
837	*/
838	static int hmR0VmxStructsAlloc(PVM pVM)
839	{
840	/*
841	* Initialize members up-front so we can cleanup properly on allocation failure.
842	*/
843	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
844	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
845	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
846	pVM->hm.s.vmx.HCPhys##a_Name = 0;
847
848	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
849	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
850	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
851	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
852
853	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
854	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
855	#endif
856	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
857
858	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
859	for (VMCPUID i = 0; i < pVM->cCpus; i++)
860	{
861	PVMCPU pVCpu = &pVM->aCpus[i];
862	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
863	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
864	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
865	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
866	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
867	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
868	#endif
869	}
870	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
871	#undef VMXLOCAL_INIT_VM_MEMOBJ
872
873	/*
874	* Allocate all the VT-x structures.
875	*/
876	int rc = VINF_SUCCESS;
877	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
878	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
879	if (RT_FAILURE(rc))
880	goto cleanup;
881	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
882	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
883	#endif
884
885	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
886	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
887	{
888	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
889	&pVM->hm.s.vmx.HCPhysApicAccess);
890	if (RT_FAILURE(rc))
891	goto cleanup;
892	}
893
894	/*
895	* Initialize per-VCPU VT-x structures.
896	*/
897	for (VMCPUID i = 0; i < pVM->cCpus; i++)
898	{
899	PVMCPU pVCpu = &pVM->aCpus[i];
900	AssertPtr(pVCpu);
901
902	/* Allocate the VM control structure (VMCS). */
903	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
904	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
905	if (RT_FAILURE(rc))
906	goto cleanup;
907
908	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
909	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
910	{
911	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
912	&pVCpu->hm.s.vmx.HCPhysVirtApic);
913	if (RT_FAILURE(rc))
914	goto cleanup;
915	}
916
917	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
918	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
919	{
920	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
921	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
922	if (RT_FAILURE(rc))
923	goto cleanup;
924	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
925	}
926
927	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
928	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
929	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
930	if (RT_FAILURE(rc))
931	goto cleanup;
932
933	/* Allocate the VM-exit MSR-load page for the host MSRs. */
934	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
935	if (RT_FAILURE(rc))
936	goto cleanup;
937	#endif
938	}
939
940	return VINF_SUCCESS;
941
942	cleanup:
943	hmR0VmxStructsFree(pVM);
944	return rc;
945	}
946
947
948	/**
949	* Does global VT-x initialization (called during module initialization).
950	*
951	* @returns VBox status code.
952	*/
953	VMMR0DECL(int) VMXR0GlobalInit(void)
954	{
955	#ifdef HMVMX_USE_FUNCTION_TABLE
956	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
957	# ifdef VBOX_STRICT
958	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
959	Assert(g_apfnVMExitHandlers[i]);
960	# endif
961	#endif
962	return VINF_SUCCESS;
963	}
964
965
966	/**
967	* Does global VT-x termination (called during module termination).
968	*/
969	VMMR0DECL(void) VMXR0GlobalTerm()
970	{
971	/* Nothing to do currently. */
972	}
973
974
975	/**
976	* Sets up and activates VT-x on the current CPU.
977	*
978	* @returns VBox status code.
979	* @param pCpu Pointer to the global CPU info struct.
980	* @param pVM Pointer to the VM (can be NULL after a host resume
981	* operation).
982	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
983	* fEnabledByHost is true).
984	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
985	* @a fEnabledByHost is true).
986	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
987	* enable VT-x on the host.
988	*/
989	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
990	{
991	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
992	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
993
994	if (!fEnabledByHost)
995	{
996	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
997	if (RT_FAILURE(rc))
998	return rc;
999	}
1000
1001	/*
1002	* Flush all EPTP tagged-TLB entries (in case any other hypervisor have been using EPTPs) so that
1003	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
1004	* each time while reusing a VPID after hitting the MaxASID limit once.
1005	*/
1006	if ( pVM
1007	&& pVM->hm.s.fNestedPaging)
1008	{
1009	/* We require ALL_CONTEXT flush-type to be available on the CPU. See hmR0VmxSetupTaggedTlb(). */
1010	Assert(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS);
1011	hmR0VmxFlushEpt(pVM, NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1012	pCpu->fFlushAsidBeforeUse = false;
1013	}
1014	else
1015	{
1016	/** @todo This is still not perfect. If on host resume (pVM is NULL or a VM
1017	* without Nested Paging triggered this function) we still have the risk
1018	* of potentially running with stale TLB-entries from other hypervisors
1019	* when later we use a VM with NestedPaging. To fix this properly we will
1020	* have to pass '&g_HvmR0' (see HMR0.cpp) to this function and read
1021	* 'vmx_ept_vpid_caps' from it. Sigh. */
1022	pCpu->fFlushAsidBeforeUse = true;
1023	}
1024
1025	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1026	++pCpu->cTlbFlushes;
1027
1028	return VINF_SUCCESS;
1029	}
1030
1031
1032	/**
1033	* Deactivates VT-x on the current CPU.
1034	*
1035	* @returns VBox status code.
1036	* @param pCpu Pointer to the global CPU info struct.
1037	* @param pvCpuPage Pointer to the VMXON region.
1038	* @param HCPhysCpuPage Physical address of the VMXON region.
1039	*
1040	* @remarks This function should never be called when SUPR0EnableVTx() or
1041	* similar was used to enable VT-x on the host.
1042	*/
1043	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1044	{
1045	NOREF(pCpu);
1046	NOREF(pvCpuPage);
1047	NOREF(HCPhysCpuPage);
1048
1049	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1050	return hmR0VmxLeaveRootMode();
1051	}
1052
1053
1054	/**
1055	* Sets the permission bits for the specified MSR in the MSR bitmap.
1056	*
1057	* @param pVCpu Pointer to the VMCPU.
1058	* @param uMSR The MSR value.
1059	* @param enmRead Whether reading this MSR causes a VM-exit.
1060	* @param enmWrite Whether writing this MSR causes a VM-exit.
1061	*/
1062	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1063	{
1064	int32_t iBit;
1065	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1066
1067	/*
1068	* Layout:
1069	* 0x000 - 0x3ff - Low MSR read bits
1070	* 0x400 - 0x7ff - High MSR read bits
1071	* 0x800 - 0xbff - Low MSR write bits
1072	* 0xc00 - 0xfff - High MSR write bits
1073	*/
1074	if (uMsr <= 0x00001FFF)
1075	iBit = uMsr;
1076	else if ( uMsr >= 0xC0000000
1077	&& uMsr <= 0xC0001FFF)
1078	{
1079	iBit = (uMsr - 0xC0000000);
1080	pbMsrBitmap += 0x400;
1081	}
1082	else
1083	{
1084	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1085	return;
1086	}
1087
1088	Assert(iBit <= 0x1fff);
1089	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1090	ASMBitSet(pbMsrBitmap, iBit);
1091	else
1092	ASMBitClear(pbMsrBitmap, iBit);
1093
1094	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1095	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1096	else
1097	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1098	}
1099
1100
1101	/**
1102	* Flushes the TLB using EPT.
1103	*
1104	* @returns VBox status code.
1105	* @param pVM Pointer to the VM.
1106	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1107	* enmFlush).
1108	* @param enmFlush Type of flush.
1109	*/
1110	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1111	{
1112	AssertPtr(pVM);
1113	Assert(pVM->hm.s.fNestedPaging);
1114
1115	uint64_t descriptor[2];
1116	if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1117	descriptor[0] = 0;
1118	else
1119	{
1120	Assert(pVCpu);
1121	descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1122	}
1123	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1124
1125	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1126	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1127	rc));
1128	if ( RT_SUCCESS(rc)
1129	&& pVCpu)
1130	{
1131	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1132	}
1133	}
1134
1135
1136	/**
1137	* Flushes the TLB using VPID.
1138	*
1139	* @returns VBox status code.
1140	* @param pVM Pointer to the VM.
1141	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1142	* enmFlush).
1143	* @param enmFlush Type of flush.
1144	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1145	* on @a enmFlush).
1146	*/
1147	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1148	{
1149	AssertPtr(pVM);
1150	Assert(pVM->hm.s.vmx.fVpid);
1151
1152	uint64_t descriptor[2];
1153	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1154	{
1155	descriptor[0] = 0;
1156	descriptor[1] = 0;
1157	}
1158	else
1159	{
1160	AssertPtr(pVCpu);
1161	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1162	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1163	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1164	descriptor[1] = GCPtr;
1165	}
1166
1167	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1168	AssertMsg(rc == VINF_SUCCESS,
1169	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1170	if ( RT_SUCCESS(rc)
1171	&& pVCpu)
1172	{
1173	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1174	}
1175	}
1176
1177
1178	/**
1179	* Invalidates a guest page by guest virtual address. Only relevant for
1180	* EPT/VPID, otherwise there is nothing really to invalidate.
1181	*
1182	* @returns VBox status code.
1183	* @param pVM Pointer to the VM.
1184	* @param pVCpu Pointer to the VMCPU.
1185	* @param GCVirt Guest virtual address of the page to invalidate.
1186	*/
1187	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1188	{
1189	AssertPtr(pVM);
1190	AssertPtr(pVCpu);
1191	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1192
1193	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1194	if (!fFlushPending)
1195	{
1196	/*
1197	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1198	* See @bugref{6043} and @bugref{6177}.
1199	*
1200	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1201	* function maybe called in a loop with individual addresses.
1202	*/
1203	if (pVM->hm.s.vmx.fVpid)
1204	{
1205	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1206	{
1207	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1208	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1209	}
1210	else
1211	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1212	}
1213	else if (pVM->hm.s.fNestedPaging)
1214	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1215	}
1216
1217	return VINF_SUCCESS;
1218	}
1219
1220
1221	/**
1222	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1223	* otherwise there is nothing really to invalidate.
1224	*
1225	* @returns VBox status code.
1226	* @param pVM Pointer to the VM.
1227	* @param pVCpu Pointer to the VMCPU.
1228	* @param GCPhys Guest physical address of the page to invalidate.
1229	*/
1230	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1231	{
1232	LogFlowFunc(("%RGp\n", GCPhys));
1233
1234	/*
1235	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1236	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1237	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1238	*/
1239	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1240	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1241	return VINF_SUCCESS;
1242	}
1243
1244
1245	/**
1246	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1247	* case where neither EPT nor VPID is supported by the CPU.
1248	*
1249	* @param pVM Pointer to the VM.
1250	* @param pVCpu Pointer to the VMCPU.
1251	*
1252	* @remarks Called with interrupts disabled.
1253	*/
1254	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1255	{
1256	NOREF(pVM);
1257	AssertPtr(pVCpu);
1258	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1259	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1260
1261	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1262	AssertPtr(pCpu);
1263
1264	pVCpu->hm.s.TlbShootdown.cPages = 0;
1265	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1266	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1267	pVCpu->hm.s.fForceTLBFlush = false;
1268	return;
1269	}
1270
1271
1272	/**
1273	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1274	*
1275	* @param pVM Pointer to the VM.
1276	* @param pVCpu Pointer to the VMCPU.
1277	* @remarks All references to "ASID" in this function pertains to "VPID" in
1278	* Intel's nomenclature. The reason is, to avoid confusion in compare
1279	* statements since the host-CPU copies are named "ASID".
1280	*
1281	* @remarks Called with interrupts disabled.
1282	*/
1283	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1284	{
1285	#ifdef VBOX_WITH_STATISTICS
1286	bool fTlbFlushed = false;
1287	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1288	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1289	if (!fTlbFlushed) \
1290	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1291	} while (0)
1292	#else
1293	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1294	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1295	#endif
1296
1297	AssertPtr(pVM);
1298	AssertPtr(pVCpu);
1299	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1300	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1301	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1302
1303	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1304	AssertPtr(pCpu);
1305
1306	/*
1307	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1308	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1309	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1310	*/
1311	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1312	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1313	{
1314	++pCpu->uCurrentAsid;
1315	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1316	{
1317	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1318	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1319	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1320	}
1321
1322	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1323	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1324	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1325
1326	/*
1327	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1328	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1329	*/
1330	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1331	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1332	HMVMX_SET_TAGGED_TLB_FLUSHED();
1333	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1334	}
1335
1336	/* Check for explicit TLB shootdowns. */
1337	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1338	{
1339	/*
1340	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1341	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1342	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1343	* but not guest-physical mappings.
1344	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1345	*/
1346	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1347	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1348	HMVMX_SET_TAGGED_TLB_FLUSHED();
1349	}
1350
1351	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1352	* not be executed. See hmQueueInvlPage() where it is commented
1353	* out. Support individual entry flushing someday. */
1354	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1355	{
1356	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1357
1358	/*
1359	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1360	* as supported by the CPU.
1361	*/
1362	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1363	{
1364	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1365	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1366	}
1367	else
1368	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1369
1370	HMVMX_SET_TAGGED_TLB_FLUSHED();
1371	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1372	}
1373
1374	pVCpu->hm.s.TlbShootdown.cPages = 0;
1375	pVCpu->hm.s.fForceTLBFlush = false;
1376
1377	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1378
1379	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1380	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1381	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1382	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1383	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1384	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1385	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1386	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1387
1388	/* Update VMCS with the VPID. */
1389	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1390	AssertRC(rc);
1391
1392	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1393	}
1394
1395
1396	/**
1397	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1398	*
1399	* @returns VBox status code.
1400	* @param pVM Pointer to the VM.
1401	* @param pVCpu Pointer to the VMCPU.
1402	*
1403	* @remarks Called with interrupts disabled.
1404	*/
1405	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1406	{
1407	AssertPtr(pVM);
1408	AssertPtr(pVCpu);
1409	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1410	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1411
1412	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1413	AssertPtr(pCpu);
1414
1415	/*
1416	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1417	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1418	*/
1419	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1420	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1421	{
1422	pVCpu->hm.s.fForceTLBFlush = true;
1423	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1424	}
1425
1426	/* Check for explicit TLB shootdown flushes. */
1427	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1428	{
1429	pVCpu->hm.s.fForceTLBFlush = true;
1430	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1431	}
1432
1433	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1434	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1435
1436	if (pVCpu->hm.s.fForceTLBFlush)
1437	{
1438	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1439	pVCpu->hm.s.fForceTLBFlush = false;
1440	}
1441	else
1442	{
1443	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1444	* not be executed. See hmQueueInvlPage() where it is commented
1445	* out. Support individual entry flushing someday. */
1446	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1447	{
1448	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1449	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1450	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1451	}
1452	else
1453	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1454	}
1455
1456	pVCpu->hm.s.TlbShootdown.cPages = 0;
1457	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1458	}
1459
1460
1461	/**
1462	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1463	*
1464	* @returns VBox status code.
1465	* @param pVM Pointer to the VM.
1466	* @param pVCpu Pointer to the VMCPU.
1467	*
1468	* @remarks Called with interrupts disabled.
1469	*/
1470	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1471	{
1472	AssertPtr(pVM);
1473	AssertPtr(pVCpu);
1474	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1475	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1476
1477	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1478
1479	/*
1480	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1481	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1482	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1483	*/
1484	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1485	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1486	{
1487	pVCpu->hm.s.fForceTLBFlush = true;
1488	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1489	}
1490
1491	/* Check for explicit TLB shootdown flushes. */
1492	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1493	{
1494	/*
1495	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1496	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1497	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1498	*/
1499	pVCpu->hm.s.fForceTLBFlush = true;
1500	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1501	}
1502
1503	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1504	if (pVCpu->hm.s.fForceTLBFlush)
1505	{
1506	++pCpu->uCurrentAsid;
1507	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1508	{
1509	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1510	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1511	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1512	}
1513
1514	pVCpu->hm.s.fForceTLBFlush = false;
1515	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1516	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1517	if (pCpu->fFlushAsidBeforeUse)
1518	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1519	}
1520	else
1521	{
1522	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1523	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1524	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1525	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1526
1527	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1528	* not be executed. See hmQueueInvlPage() where it is commented
1529	* out. Support individual entry flushing someday. */
1530	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1531	{
1532	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1533	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1534	{
1535	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1536	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1537	}
1538	else
1539	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1540	}
1541	else
1542	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1543	}
1544
1545	pVCpu->hm.s.TlbShootdown.cPages = 0;
1546	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1547
1548	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1549	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1550	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1551	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1552	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1553	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1554
1555	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1556	AssertRC(rc);
1557	}
1558
1559
1560	/**
1561	* Flushes the guest TLB entry based on CPU capabilities.
1562	*
1563	* @param pVCpu Pointer to the VMCPU.
1564	*/
1565	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1566	{
1567	PVM pVM = pVCpu->CTX_SUFF(pVM);
1568	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1569	{
1570	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1571	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1572	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1573	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1574	default:
1575	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1576	break;
1577	}
1578	}
1579
1580
1581	/**
1582	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1583	* TLB entries from the host TLB before VM-entry.
1584	*
1585	* @returns VBox status code.
1586	* @param pVM Pointer to the VM.
1587	*/
1588	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1589	{
1590	/*
1591	* Determine optimal flush type for Nested Paging.
1592	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1593	* guest execution (see hmR3InitFinalizeR0()).
1594	*/
1595	if (pVM->hm.s.fNestedPaging)
1596	{
1597	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1598	{
1599	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1600	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1601	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1602	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1603	else
1604	{
1605	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1606	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1607	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1608	}
1609
1610	/* Make sure the write-back cacheable memory type for EPT is supported. */
1611	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1612	{
1613	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1614	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1615	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1616	}
1617	}
1618	else
1619	{
1620	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1621	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1622	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1623	}
1624	}
1625
1626	/*
1627	* Determine optimal flush type for VPID.
1628	*/
1629	if (pVM->hm.s.vmx.fVpid)
1630	{
1631	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1632	{
1633	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1634	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1635	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1636	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1637	else
1638	{
1639	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1640	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1641	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1642	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1643	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1644	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1645	pVM->hm.s.vmx.fVpid = false;
1646	}
1647	}
1648	else
1649	{
1650	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1651	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1652	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1653	pVM->hm.s.vmx.fVpid = false;
1654	}
1655	}
1656
1657	/*
1658	* Setup the handler for flushing tagged-TLBs.
1659	*/
1660	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1661	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1662	else if (pVM->hm.s.fNestedPaging)
1663	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1664	else if (pVM->hm.s.vmx.fVpid)
1665	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1666	else
1667	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1668	return VINF_SUCCESS;
1669	}
1670
1671
1672	/**
1673	* Sets up pin-based VM-execution controls in the VMCS.
1674	*
1675	* @returns VBox status code.
1676	* @param pVM Pointer to the VM.
1677	* @param pVCpu Pointer to the VMCPU.
1678	*/
1679	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1680	{
1681	AssertPtr(pVM);
1682	AssertPtr(pVCpu);
1683
1684	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1685	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1686
1687	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1688	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1689	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1690
1691	/* Enable the VMX preemption timer. */
1692	if (pVM->hm.s.vmx.fUsePreemptTimer)
1693	{
1694	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1695	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1696	}
1697
1698	if ((val & zap) != val)
1699	{
1700	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1701	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1702	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1703	}
1704
1705	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1706	AssertRCReturn(rc, rc);
1707
1708	/* Update VCPU with the currently set pin-based VM-execution controls. */
1709	pVCpu->hm.s.vmx.u32PinCtls = val;
1710	return rc;
1711	}
1712
1713
1714	/**
1715	* Sets up processor-based VM-execution controls in the VMCS.
1716	*
1717	* @returns VBox status code.
1718	* @param pVM Pointer to the VM.
1719	* @param pVMCPU Pointer to the VMCPU.
1720	*/
1721	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1722	{
1723	AssertPtr(pVM);
1724	AssertPtr(pVCpu);
1725
1726	int rc = VERR_INTERNAL_ERROR_5;
1727	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1728	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1729
1730	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1731	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1732	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1733	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1734	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1735	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1736	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1737
1738	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1739	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1740	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1741	{
1742	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1743	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1744	}
1745
1746	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1747	if (!pVM->hm.s.fNestedPaging)
1748	{
1749	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1750	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
1751	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
1752	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
1753	}
1754
1755	/* Use TPR shadowing if supported by the CPU. */
1756	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1757	{
1758	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1759	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1760	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1761	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1762	AssertRCReturn(rc, rc);
1763
1764	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1765	/* CR8 writes causes a VM-exit based on TPR threshold. */
1766	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
1767	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
1768	}
1769	else
1770	{
1771	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1772	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1773	}
1774
1775	/* Use MSR-bitmaps if supported by the CPU. */
1776	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1777	{
1778	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
1779
1780	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1781	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1782	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1783	AssertRCReturn(rc, rc);
1784
1785	/*
1786	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1787	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1788	*/
1789	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1790	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1791	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1792	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1793	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1794	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1795	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1796	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1797	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1798	}
1799
1800	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1801	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1802	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1803
1804	if ((val & zap) != val)
1805	{
1806	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1807	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1808	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1809	}
1810
1811	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
1812	AssertRCReturn(rc, rc);
1813
1814	/* Update VCPU with the currently set processor-based VM-execution controls. */
1815	pVCpu->hm.s.vmx.u32ProcCtls = val;
1816
1817	/*
1818	* Secondary processor-based VM-execution controls.
1819	*/
1820	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1821	{
1822	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1823	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1824
1825	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1826	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1827
1828	if (pVM->hm.s.fNestedPaging)
1829	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1830	else
1831	{
1832	/*
1833	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1834	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
1835	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1836	*/
1837	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1838	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1839	}
1840
1841	if (pVM->hm.s.vmx.fVpid)
1842	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1843
1844	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1845	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1846
1847	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1848	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1849	* done dynamically. */
1850	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1851	{
1852	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1853	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1854	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1855	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1856	AssertRCReturn(rc, rc);
1857	}
1858
1859	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1860	{
1861	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1862	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1863	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1864	}
1865
1866	if ((val & zap) != val)
1867	{
1868	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1869	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1870	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1871	}
1872
1873	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
1874	AssertRCReturn(rc, rc);
1875
1876	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1877	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1878	}
1879
1880	return VINF_SUCCESS;
1881	}
1882
1883
1884	/**
1885	* Sets up miscellaneous (everything other than Pin & Processor-based
1886	* VM-execution) control fields in the VMCS.
1887	*
1888	* @returns VBox status code.
1889	* @param pVM Pointer to the VM.
1890	* @param pVCpu Pointer to the VMCPU.
1891	*/
1892	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1893	{
1894	AssertPtr(pVM);
1895	AssertPtr(pVCpu);
1896
1897	int rc = VERR_GENERAL_FAILURE;
1898
1899	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1900	#if 0
1901	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1902	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1903	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1904
1905	/*
1906	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1907	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1908	* We thus use the exception bitmap to control it rather than use both.
1909	*/
1910	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1911	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1912
1913	/** @todo Explore possibility of using IO-bitmaps. */
1914	/* All IO & IOIO instructions cause VM-exits. */
1915	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1916	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1917
1918	/* Initialize the MSR-bitmap area. */
1919	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1920	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
1921	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1922	#endif
1923
1924	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1925	/* Setup MSR autoloading/storing. */
1926	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1927	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1928	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1929	AssertRCReturn(rc, rc);
1930	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1931	AssertRCReturn(rc, rc);
1932
1933	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1934	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1935	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1936	AssertRCReturn(rc, rc);
1937	#endif
1938
1939	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1940	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1941	AssertRCReturn(rc, rc);
1942
1943	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1944	#if 0
1945	/* Setup debug controls */
1946	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
1947	AssertRCReturn(rc, rc);
1948	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1949	AssertRCReturn(rc, rc);
1950	#endif
1951
1952	return rc;
1953	}
1954
1955
1956	/**
1957	* Sets up the initial exception bitmap in the VMCS based on static conditions
1958	* (i.e. conditions that cannot ever change at runtime).
1959	*
1960	* @returns VBox status code.
1961	* @param pVM Pointer to the VM.
1962	* @param pVCpu Pointer to the VMCPU.
1963	*/
1964	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1965	{
1966	AssertPtr(pVM);
1967	AssertPtr(pVCpu);
1968
1969	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1970
1971	uint32_t u32XcptBitmap = 0;
1972
1973	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1974	if (!pVM->hm.s.fNestedPaging)
1975	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1976
1977	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1978	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1979	AssertRCReturn(rc, rc);
1980	return rc;
1981	}
1982
1983
1984	/**
1985	* Sets up the initial guest-state mask. The guest-state mask is consulted
1986	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1987	* for the nested virtualization case (as it would cause a VM-exit).
1988	*
1989	* @param pVCpu Pointer to the VMCPU.
1990	*/
1991	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1992	{
1993	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1994	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
1995	return VINF_SUCCESS;
1996	}
1997
1998
1999	/**
2000	* Does per-VM VT-x initialization.
2001	*
2002	* @returns VBox status code.
2003	* @param pVM Pointer to the VM.
2004	*/
2005	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2006	{
2007	LogFlowFunc(("pVM=%p\n", pVM));
2008
2009	int rc = hmR0VmxStructsAlloc(pVM);
2010	if (RT_FAILURE(rc))
2011	{
2012	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2013	return rc;
2014	}
2015
2016	return VINF_SUCCESS;
2017	}
2018
2019
2020	/**
2021	* Does per-VM VT-x termination.
2022	*
2023	* @returns VBox status code.
2024	* @param pVM Pointer to the VM.
2025	*/
2026	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2027	{
2028	LogFlowFunc(("pVM=%p\n", pVM));
2029
2030	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2031	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2032	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2033	#endif
2034	hmR0VmxStructsFree(pVM);
2035	return VINF_SUCCESS;
2036	}
2037
2038
2039	/**
2040	* Sets up the VM for execution under VT-x.
2041	* This function is only called once per-VM during initialization.
2042	*
2043	* @returns VBox status code.
2044	* @param pVM Pointer to the VM.
2045	*/
2046	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2047	{
2048	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2049	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2050
2051	LogFlowFunc(("pVM=%p\n", pVM));
2052
2053	/*
2054	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2055	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2056	*/
2057	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2058	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2059	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2060	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2061	{
2062	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2063	return VERR_INTERNAL_ERROR;
2064	}
2065
2066	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2067	/*
2068	* This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2069	* the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2070	*/
2071	if ( pVM->hm.s.fAllow64BitGuests
2072	&& !HMVMX_IS_64BIT_HOST_MODE())
2073	{
2074	LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2075	return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2076	}
2077	#endif
2078
2079	/* Initialize these always, see hmR3InitFinalizeR0().*/
2080	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2081	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2082
2083	/* Setup the tagged-TLB flush handlers. */
2084	int rc = hmR0VmxSetupTaggedTlb(pVM);
2085	if (RT_FAILURE(rc))
2086	{
2087	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2088	return rc;
2089	}
2090
2091	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2092	{
2093	PVMCPU pVCpu = &pVM->aCpus[i];
2094	AssertPtr(pVCpu);
2095	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2096
2097	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2098	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2099
2100	/* Set revision dword at the beginning of the VMCS structure. */
2101	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
2102
2103	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2104	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2105	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2106	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2107
2108	/* Load this VMCS as the current VMCS. */
2109	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2110	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2111	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2112
2113	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2114	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2115	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2116
2117	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2118	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2119	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2120
2121	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2122	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2123	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2124
2125	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2126	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2127	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2128
2129	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2130	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2131	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2132
2133	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2134	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2135	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2136	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2137	#endif
2138
2139	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2140	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2141	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2142	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2143
2144	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2145	}
2146
2147	return VINF_SUCCESS;
2148	}
2149
2150
2151	/**
2152	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2153	* the VMCS.
2154	*
2155	* @returns VBox status code.
2156	* @param pVM Pointer to the VM.
2157	* @param pVCpu Pointer to the VMCPU.
2158	*/
2159	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2160	{
2161	RTCCUINTREG uReg = ASMGetCR0();
2162	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2163	AssertRCReturn(rc, rc);
2164
2165	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2166	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2167	if (HMVMX_IS_64BIT_HOST_MODE())
2168	{
2169	uint64_t uRegCR3 = HMR0Get64bitCR3();
2170	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2171	}
2172	else
2173	#endif
2174	{
2175	uReg = ASMGetCR3();
2176	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2177	}
2178	AssertRCReturn(rc, rc);
2179
2180	uReg = ASMGetCR4();
2181	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2182	AssertRCReturn(rc, rc);
2183	return rc;
2184	}
2185
2186
2187	/**
2188	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2189	* the host-state area in the VMCS.
2190	*
2191	* @returns VBox status code.
2192	* @param pVM Pointer to the VM.
2193	* @param pVCpu Pointer to the VMCPU.
2194	*/
2195	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2196	{
2197	int rc = VERR_INTERNAL_ERROR_5;
2198	RTSEL uSelDS = 0;
2199	RTSEL uSelES = 0;
2200	RTSEL uSelFS = 0;
2201	RTSEL uSelGS = 0;
2202	RTSEL uSelTR = 0;
2203
2204	/*
2205	* Host DS, ES, FS and GS segment registers.
2206	*/
2207	#if HC_ARCH_BITS == 64
2208	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2209	uSelDS = ASMGetDS();
2210	uSelES = ASMGetES();
2211	uSelFS = ASMGetFS();
2212	uSelGS = ASMGetGS();
2213	#endif
2214
2215	/*
2216	* Host CS and SS segment registers.
2217	*/
2218	RTSEL uSelCS;
2219	RTSEL uSelSS;
2220	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2221	if (HMVMX_IS_64BIT_HOST_MODE())
2222	{
2223	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2224	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2225	}
2226	else
2227	{
2228	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2229	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2230	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2231	}
2232	#else
2233	uSelCS = ASMGetCS();
2234	uSelSS = ASMGetSS();
2235	#endif
2236
2237	/*
2238	* Host TR segment register.
2239	*/
2240	uSelTR = ASMGetTR();
2241
2242	#if HC_ARCH_BITS == 64
2243	/*
2244	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2245	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2246	*/
2247	if (uSelDS & (X86_SEL_RPL \| X86_SEL_LDT))
2248	{
2249	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_DS;
2250	pVCpu->hm.s.vmx.RestoreHost.uHostSelDS = uSelDS;
2251	uSelDS = 0;
2252	}
2253	if (uSelES & (X86_SEL_RPL \| X86_SEL_LDT))
2254	{
2255	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_ES;
2256	pVCpu->hm.s.vmx.RestoreHost.uHostSelES = uSelES;
2257	uSelES = 0;
2258	}
2259	if (uSelFS & (X86_SEL_RPL \| X86_SEL_LDT))
2260	{
2261	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_FS;
2262	pVCpu->hm.s.vmx.RestoreHost.uHostSelFS = uSelFS;
2263	uSelFS = 0;
2264	}
2265	if (uSelGS & (X86_SEL_RPL \| X86_SEL_LDT))
2266	{
2267	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_GS;
2268	pVCpu->hm.s.vmx.RestoreHost.uHostSelGS = uSelGS;
2269	uSelGS = 0;
2270	}
2271	#endif
2272
2273	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2274	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2275	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2276	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2277	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2278	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2279	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2280	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2281	Assert(uSelCS);
2282	Assert(uSelTR);
2283
2284	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2285	#if 0
2286	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2287	Assert(uSelSS != 0);
2288	#endif
2289
2290	/* Write these host selector fields into the host-state area in the VMCS. */
2291	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2292	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2293	#if HC_ARCH_BITS == 64
2294	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2295	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2296	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2297	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2298	#endif
2299	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2300
2301	/*
2302	* Host GDTR and IDTR.
2303	*/
2304	RTGDTR Gdtr;
2305	RT_ZERO(Gdtr);
2306	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2307	if (HMVMX_IS_64BIT_HOST_MODE())
2308	{
2309	X86XDTR64 Gdtr64;
2310	X86XDTR64 Idtr64;
2311	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2312	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2313	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2314
2315	Gdtr.cbGdt = Gdtr64.cb;
2316	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2317	}
2318	else
2319	#endif
2320	{
2321	RTIDTR Idtr;
2322	ASMGetGDTR(&Gdtr);
2323	ASMGetIDTR(&Idtr);
2324	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2325	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2326
2327	#if HC_ARCH_BITS == 64
2328	/*
2329	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2330	* maximum limit (0xffff) on every VM-exit.
2331	*/
2332	if (Gdtr.cbGdt != 0xffff)
2333	{
2334	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2335	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2336	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2337	}
2338
2339	/*
2340	* IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2341	* is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2342	* Intel spec. 6.2 "Exception and Interrupt Vectors".
2343	*/
2344	if (Idtr.cbIdt < 0x0fff)
2345	{
2346	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2347	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2348	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2349	}
2350	#endif
2351	}
2352
2353	/*
2354	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2355	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2356	*/
2357	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2358	{
2359	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2360	return VERR_VMX_INVALID_HOST_STATE;
2361	}
2362
2363	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2364	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2365	if (HMVMX_IS_64BIT_HOST_MODE())
2366	{
2367	/* We need the 64-bit TR base for hybrid darwin. */
2368	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2369	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2370	}
2371	else
2372	#endif
2373	{
2374	uintptr_t uTRBase;
2375	#if HC_ARCH_BITS == 64
2376	uTRBase = X86DESC64_BASE(pDesc);
2377
2378	/*
2379	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2380	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2381	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2382	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2383	*
2384	* [1] See Intel spec. 3.5 "System Descriptor Types".
2385	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2386	*/
2387	Assert(pDesc->System.u4Type == 11);
2388	if ( pDesc->System.u16LimitLow != 0x67
2389	\|\| pDesc->System.u4LimitHigh)
2390	{
2391	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
2392	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2393
2394	/* Store the GDTR here as we need it while restoring TR. */
2395	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2396	}
2397	#else
2398	uTRBase = X86DESC_BASE(pDesc);
2399	#endif
2400	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2401	}
2402	AssertRCReturn(rc, rc);
2403
2404	/*
2405	* Host FS base and GS base.
2406	*/
2407	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2408	if (HMVMX_IS_64BIT_HOST_MODE())
2409	{
2410	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2411	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2412	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2413	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2414
2415	# if HC_ARCH_BITS == 64
2416	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2417	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2418	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2419	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2420	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2421	# endif
2422	}
2423	#endif
2424	return rc;
2425	}
2426
2427
2428	/**
2429	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2430	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2431	* the host after every successful VM exit.
2432	*
2433	* @returns VBox status code.
2434	* @param pVM Pointer to the VM.
2435	* @param pVCpu Pointer to the VMCPU.
2436	*/
2437	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2438	{
2439	AssertPtr(pVCpu);
2440	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2441
2442	int rc = VINF_SUCCESS;
2443	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2444	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2445	uint32_t cHostMsrs = 0;
2446	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2447
2448	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2449	{
2450	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2451
2452	# if HC_ARCH_BITS == 64
2453	/* Paranoia. 64-bit code requires these bits to be set always. */
2454	Assert((u64HostEfer & (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME));
2455
2456	/*
2457	* We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2458	* All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2459	* some reason (e.g. allow transparent reads) we would activate the code below.
2460	*/
2461	# if 0
2462	/* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2463	Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2464	/* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2465	See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2466	if (CPUMIsGuestInLongMode(pVCpu))
2467	{
2468	uint64_t u64GuestEfer;
2469	rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2470	AssertRC(rc);
2471
2472	if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2473	{
2474	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2475	pHostMsr->u32Reserved = 0;
2476	pHostMsr->u64Value = u64HostEfer;
2477	pHostMsr++; cHostMsrs++;
2478	}
2479	}
2480	# endif
2481	# else /* HC_ARCH_BITS != 64 */
2482	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2483	pHostMsr->u32Reserved = 0;
2484	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2485	if (CPUMIsGuestInLongMode(pVCpu))
2486	{
2487	/* Must match the EFER value in our 64 bits switcher. */
2488	pHostMsr->u64Value = u64HostEfer \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2489	}
2490	else
2491	# endif
2492	pHostMsr->u64Value = u64HostEfer;
2493	pHostMsr++; cHostMsrs++;
2494	# endif /* HC_ARCH_BITS == 64 */
2495	}
2496
2497	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2498	if (HMVMX_IS_64BIT_HOST_MODE())
2499	{
2500	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2501	pHostMsr->u32Reserved = 0;
2502	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2503	pHostMsr++; cHostMsrs++;
2504	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2505	pHostMsr->u32Reserved = 0;
2506	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2507	pHostMsr++; cHostMsrs++;
2508	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2509	pHostMsr->u32Reserved = 0;
2510	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2511	pHostMsr++; cHostMsrs++;
2512	pHostMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2513	pHostMsr->u32Reserved = 0;
2514	pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2515	pHostMsr++; cHostMsrs++;
2516	}
2517	# endif
2518
2519	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2520	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2521	{
2522	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2523	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2524	}
2525
2526	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2527	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2528
2529	/*
2530	* Host Sysenter MSRs.
2531	*/
2532	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2533	AssertRCReturn(rc, rc);
2534	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2535	if (HMVMX_IS_64BIT_HOST_MODE())
2536	{
2537	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2538	AssertRCReturn(rc, rc);
2539	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2540	}
2541	else
2542	{
2543	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2544	AssertRCReturn(rc, rc);
2545	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2546	}
2547	#elif HC_ARCH_BITS == 32
2548	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2549	AssertRCReturn(rc, rc);
2550	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2551	#else
2552	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2553	AssertRCReturn(rc, rc);
2554	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2555	#endif
2556	AssertRCReturn(rc, rc);
2557
2558	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2559	* hmR0VmxSetupExitCtls() !! */
2560	return rc;
2561	}
2562
2563
2564	/**
2565	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2566	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2567	* controls".
2568	*
2569	* @returns VBox status code.
2570	* @param pVCpu Pointer to the VMCPU.
2571	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2572	* out-of-sync. Make sure to update the required fields
2573	* before using them.
2574	*
2575	* @remarks No-long-jump zone!!!
2576	*/
2577	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2578	{
2579	int rc = VINF_SUCCESS;
2580	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2581	{
2582	PVM pVM = pVCpu->CTX_SUFF(pVM);
2583	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2584	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2585
2586	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2587	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2588
2589	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2590	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2591	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2592	else
2593	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2594
2595	/*
2596	* The following should not be set (since we're not in SMM mode):
2597	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2598	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2599	*/
2600
2601	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2602	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2603	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2604
2605	if ((val & zap) != val)
2606	{
2607	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2608	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2609	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2610	}
2611
2612	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2613	AssertRCReturn(rc, rc);
2614
2615	/* Update VCPU with the currently set VM-exit controls. */
2616	pVCpu->hm.s.vmx.u32EntryCtls = val;
2617	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2618	}
2619	return rc;
2620	}
2621
2622
2623	/**
2624	* Sets up the VM-exit controls in the VMCS.
2625	*
2626	* @returns VBox status code.
2627	* @param pVM Pointer to the VM.
2628	* @param pVCpu Pointer to the VMCPU.
2629	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2630	* out-of-sync. Make sure to update the required fields
2631	* before using them.
2632	*
2633	* @remarks requires EFER.
2634	*/
2635	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2636	{
2637	int rc = VINF_SUCCESS;
2638	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2639	{
2640	PVM pVM = pVCpu->CTX_SUFF(pVM);
2641	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2642	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2643
2644	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2645	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2646
2647	/*
2648	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2649	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2650	*/
2651	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2652	if (HMVMX_IS_64BIT_HOST_MODE())
2653	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2654	else
2655	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2656	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2657	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2658	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2659	else
2660	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2661	#endif
2662
2663	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2664	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2665
2666	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2667	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2668	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2669	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2670	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2671
2672	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2673	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2674
2675	if ((val & zap) != val)
2676	{
2677	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2678	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2679	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2680	}
2681
2682	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2683	AssertRCReturn(rc, rc);
2684
2685	/* Update VCPU with the currently set VM-exit controls. */
2686	pVCpu->hm.s.vmx.u32ExitCtls = val;
2687	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2688	}
2689	return rc;
2690	}
2691
2692
2693	/**
2694	* Loads the guest APIC and related state.
2695	*
2696	* @returns VBox status code.
2697	* @param pVM Pointer to the VM.
2698	* @param pVCpu Pointer to the VMCPU.
2699	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2700	* out-of-sync. Make sure to update the required fields
2701	* before using them.
2702	*/
2703	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2704	{
2705	int rc = VINF_SUCCESS;
2706	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2707	{
2708	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2709	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2710	{
2711	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2712
2713	bool fPendingIntr = false;
2714	uint8_t u8Tpr = 0;
2715	uint8_t u8PendingIntr = 0;
2716	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2717	AssertRCReturn(rc, rc);
2718
2719	/*
2720	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
2721	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
2722	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2723	* the interrupt when we VM-exit for other reasons.
2724	*/
2725	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2726	uint32_t u32TprThreshold = 0;
2727	if (fPendingIntr)
2728	{
2729	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2730	const uint8_t u8PendingPriority = (u8PendingIntr >> 4);
2731	const uint8_t u8TprPriority = (u8Tpr >> 4) & 7;
2732	if (u8PendingPriority <= u8TprPriority)
2733	u32TprThreshold = u8PendingPriority;
2734	else
2735	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
2736	}
2737	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2738
2739	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2740	AssertRCReturn(rc, rc);
2741	}
2742
2743	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2744	}
2745	return rc;
2746	}
2747
2748
2749	/**
2750	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2751	*
2752	* @returns Guest's interruptibility-state.
2753	* @param pVCpu Pointer to the VMCPU.
2754	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2755	* out-of-sync. Make sure to update the required fields
2756	* before using them.
2757	*
2758	* @remarks No-long-jump zone!!!
2759	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2760	*/
2761	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2762	{
2763	/*
2764	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2765	* inhibit interrupts or clear any existing interrupt-inhibition.
2766	*/
2767	uint32_t uIntrState = 0;
2768	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2769	{
2770	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2771	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2772	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2773	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2774	{
2775	/*
2776	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2777	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
2778	*/
2779	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2780	}
2781	else if (pMixedCtx->eflags.Bits.u1IF)
2782	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2783	else
2784	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2785	}
2786	return uIntrState;
2787	}
2788
2789
2790	/**
2791	* Loads the guest's interruptibility-state into the guest-state area in the
2792	* VMCS.
2793	*
2794	* @returns VBox status code.
2795	* @param pVCpu Pointer to the VMCPU.
2796	* @param uIntrState The interruptibility-state to set.
2797	*/
2798	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2799	{
2800	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2801	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2802	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2803	AssertRCReturn(rc, rc);
2804	return rc;
2805	}
2806
2807
2808	/**
2809	* Loads the guest's RIP into the guest-state area in the VMCS.
2810	*
2811	* @returns VBox status code.
2812	* @param pVCpu Pointer to the VMCPU.
2813	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2814	* out-of-sync. Make sure to update the required fields
2815	* before using them.
2816	*
2817	* @remarks No-long-jump zone!!!
2818	*/
2819	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2820	{
2821	int rc = VINF_SUCCESS;
2822	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2823	{
2824	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2825	AssertRCReturn(rc, rc);
2826	Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2827	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2828	}
2829	return rc;
2830	}
2831
2832
2833	/**
2834	* Loads the guest's RSP into the guest-state area in the VMCS.
2835	*
2836	* @returns VBox status code.
2837	* @param pVCpu Pointer to the VMCPU.
2838	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2839	* out-of-sync. Make sure to update the required fields
2840	* before using them.
2841	*
2842	* @remarks No-long-jump zone!!!
2843	*/
2844	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2845	{
2846	int rc = VINF_SUCCESS;
2847	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2848	{
2849	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2850	AssertRCReturn(rc, rc);
2851	Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
2852	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2853	}
2854	return rc;
2855	}
2856
2857
2858	/**
2859	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2860	*
2861	* @returns VBox status code.
2862	* @param pVCpu Pointer to the VMCPU.
2863	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2864	* out-of-sync. Make sure to update the required fields
2865	* before using them.
2866	*
2867	* @remarks No-long-jump zone!!!
2868	*/
2869	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2870	{
2871	int rc = VINF_SUCCESS;
2872	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2873	{
2874	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2875	Let us assert it as such and use 32-bit VMWRITE. */
2876	Assert(!(pMixedCtx->rflags.u64 >> 32));
2877	X86EFLAGS uEFlags = pMixedCtx->eflags;
2878	uEFlags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2879	uEFlags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2880
2881	/*
2882	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2883	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2884	*/
2885	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2886	{
2887	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2888	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2889	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uEFlags.u32; /* Save the original eflags of the real-mode guest. */
2890	uEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2891	uEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2892	}
2893
2894	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, uEFlags.u32);
2895	AssertRCReturn(rc, rc);
2896
2897	Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", uEFlags.u32));
2898	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2899	}
2900	return rc;
2901	}
2902
2903
2904	/**
2905	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2906	*
2907	* @returns VBox status code.
2908	* @param pVCpu Pointer to the VMCPU.
2909	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2910	* out-of-sync. Make sure to update the required fields
2911	* before using them.
2912	*
2913	* @remarks No-long-jump zone!!!
2914	*/
2915	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2916	{
2917	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2918	AssertRCReturn(rc, rc);
2919	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2920	AssertRCReturn(rc, rc);
2921	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2922	AssertRCReturn(rc, rc);
2923	return rc;
2924	}
2925
2926
2927	/**
2928	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2929	* in the VMCS.
2930	*
2931	* @returns VBox status code.
2932	* @param pVM Pointer to the VM.
2933	* @param pVCpu Pointer to the VMCPU.
2934	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2935	* out-of-sync. Make sure to update the required fields
2936	* before using them.
2937	*
2938	* @remarks No-long-jump zone!!!
2939	*/
2940	static int hmR0VmxLoadGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pCtx)
2941	{
2942	int rc = VINF_SUCCESS;
2943	PVM pVM = pVCpu->CTX_SUFF(pVM);
2944
2945	/*
2946	* Guest CR0.
2947	* Guest FPU.
2948	*/
2949	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2950	{
2951	Assert(!(pCtx->cr0 >> 32));
2952	uint32_t u32GuestCR0 = pCtx->cr0;
2953
2954	/* The guest's view (read access) of its CR0 is unblemished. */
2955	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2956	AssertRCReturn(rc, rc);
2957	Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2958
2959	/* Setup VT-x's view of the guest CR0. */
2960	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2961	if (pVM->hm.s.fNestedPaging)
2962	{
2963	if (CPUMIsGuestPagingEnabledEx(pCtx))
2964	{
2965	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2966	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2967	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
2968	}
2969	else
2970	{
2971	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2972	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2973	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2974	}
2975
2976	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
2977	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2978	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2979
2980	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
2981	AssertRCReturn(rc, rc);
2982	}
2983	else
2984	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
2985
2986	/*
2987	* Guest FPU bits.
2988	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2989	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2990	*/
2991	u32GuestCR0 \|= X86_CR0_NE;
2992	bool fInterceptNM = false;
2993	if (CPUMIsGuestFPUStateActive(pVCpu))
2994	{
2995	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2996	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2997	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2998	}
2999	else
3000	{
3001	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3002	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3003	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3004	}
3005
3006	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3007	bool fInterceptMF = false;
3008	if (!(pCtx->cr0 & X86_CR0_NE))
3009	fInterceptMF = true;
3010
3011	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3012	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3013	{
3014	Assert(PDMVmmDevHeapIsEnabled(pVM));
3015	Assert(pVM->hm.s.vmx.pRealModeTSS);
3016	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3017	fInterceptNM = true;
3018	fInterceptMF = true;
3019	}
3020	else
3021	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3022
3023	if (fInterceptNM)
3024	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3025	else
3026	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3027
3028	if (fInterceptMF)
3029	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3030	else
3031	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3032
3033	/* Additional intercepts for debugging, define these yourself explicitly. */
3034	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3035	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3036	\| RT_BIT(X86_XCPT_BP)
3037	\| RT_BIT(X86_XCPT_DB)
3038	\| RT_BIT(X86_XCPT_DE)
3039	\| RT_BIT(X86_XCPT_NM)
3040	\| RT_BIT(X86_XCPT_UD)
3041	\| RT_BIT(X86_XCPT_NP)
3042	\| RT_BIT(X86_XCPT_SS)
3043	\| RT_BIT(X86_XCPT_GP)
3044	\| RT_BIT(X86_XCPT_PF)
3045	\| RT_BIT(X86_XCPT_MF)
3046	;
3047	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3048	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3049	#endif
3050
3051	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3052
3053	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3054	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
3055	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
3056	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3057	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3058	else
3059	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3060
3061	u32GuestCR0 \|= uSetCR0;
3062	u32GuestCR0 &= uZapCR0;
3063	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3064
3065	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3066	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3067	AssertRCReturn(rc, rc);
3068	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3069	AssertRCReturn(rc, rc);
3070	Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3071
3072	/*
3073	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3074	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3075	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3076	*/
3077	uint32_t u32CR0Mask = 0;
3078	u32CR0Mask = X86_CR0_PE
3079	\| X86_CR0_NE
3080	\| X86_CR0_WP
3081	\| X86_CR0_PG
3082	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3083	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3084	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3085	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3086	u32CR0Mask &= ~X86_CR0_PE;
3087	if (pVM->hm.s.fNestedPaging)
3088	u32CR0Mask &= ~X86_CR0_WP;
3089
3090	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3091	if (fInterceptNM)
3092	u32CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
3093	else
3094	u32CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
3095
3096	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3097	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3098	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3099	AssertRCReturn(rc, rc);
3100
3101	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
3102	}
3103
3104	/*
3105	* Guest CR2.
3106	* It's always loaded in the assembler code. Nothing to do here.
3107	*/
3108
3109	/*
3110	* Guest CR3.
3111	*/
3112	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
3113	{
3114	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3115	if (pVM->hm.s.fNestedPaging)
3116	{
3117	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3118
3119	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3120	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3121	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3122	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3123
3124	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3125	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3126	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3127
3128	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3129	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3130	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3131	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3132
3133	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3134	AssertRCReturn(rc, rc);
3135	Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3136
3137	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3138	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
3139	{
3140	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3141	if (CPUMIsGuestInPAEModeEx(pCtx))
3142	{
3143	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3144	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3145	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3146	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3147	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3148	}
3149
3150	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3151	have Unrestricted Execution to handle the guest when it's not using paging. */
3152	GCPhysGuestCR3 = pCtx->cr3;
3153	}
3154	else
3155	{
3156	/*
3157	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3158	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3159	* EPT takes care of translating it to host-physical addresses.
3160	*/
3161	RTGCPHYS GCPhys;
3162	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3163	Assert(PDMVmmDevHeapIsEnabled(pVM));
3164
3165	/* We obtain it here every time as the guest could have relocated this PCI region. */
3166	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3167	AssertRCReturn(rc, rc);
3168
3169	GCPhysGuestCR3 = GCPhys;
3170	}
3171
3172	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3173	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3174	}
3175	else
3176	{
3177	/* Non-nested paging case, just use the hypervisor's CR3. */
3178	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3179
3180	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3181	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3182	}
3183	AssertRCReturn(rc, rc);
3184
3185	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
3186	}
3187
3188	/*
3189	* Guest CR4.
3190	*/
3191	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
3192	{
3193	Assert(!(pCtx->cr4 >> 32));
3194	uint32_t u32GuestCR4 = pCtx->cr4;
3195
3196	/* The guest's view of its CR4 is unblemished. */
3197	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3198	AssertRCReturn(rc, rc);
3199	Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3200
3201	/* Setup VT-x's view of the guest CR4. */
3202	/*
3203	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3204	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3205	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3206	*/
3207	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3208	{
3209	Assert(pVM->hm.s.vmx.pRealModeTSS);
3210	Assert(PDMVmmDevHeapIsEnabled(pVM));
3211	u32GuestCR4 &= ~X86_CR4_VME;
3212	}
3213
3214	if (pVM->hm.s.fNestedPaging)
3215	{
3216	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3217	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3218	{
3219	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3220	u32GuestCR4 \|= X86_CR4_PSE;
3221	/* Our identity mapping is a 32 bits page directory. */
3222	u32GuestCR4 &= ~X86_CR4_PAE;
3223	}
3224	/* else use guest CR4.*/
3225	}
3226	else
3227	{
3228	/*
3229	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3230	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3231	*/
3232	switch (pVCpu->hm.s.enmShadowMode)
3233	{
3234	case PGMMODE_REAL: /* Real-mode. */
3235	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3236	case PGMMODE_32_BIT: /* 32-bit paging. */
3237	{
3238	u32GuestCR4 &= ~X86_CR4_PAE;
3239	break;
3240	}
3241
3242	case PGMMODE_PAE: /* PAE paging. */
3243	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3244	{
3245	u32GuestCR4 \|= X86_CR4_PAE;
3246	break;
3247	}
3248
3249	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3250	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3251	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3252	break;
3253	#endif
3254	default:
3255	AssertFailed();
3256	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3257	}
3258	}
3259
3260	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3261	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3262	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3263	u32GuestCR4 \|= uSetCR4;
3264	u32GuestCR4 &= uZapCR4;
3265
3266	/* Write VT-x's view of the guest CR4 into the VMCS. */
3267	Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3268	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3269	AssertRCReturn(rc, rc);
3270
3271	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3272	uint32_t u32CR4Mask = 0;
3273	u32CR4Mask = X86_CR4_VME
3274	\| X86_CR4_PAE
3275	\| X86_CR4_PGE
3276	\| X86_CR4_PSE
3277	\| X86_CR4_VMXE;
3278	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3279	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3280	AssertRCReturn(rc, rc);
3281
3282	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3283	}
3284	return rc;
3285	}
3286
3287
3288	/**
3289	* Loads the guest debug registers into the guest-state area in the VMCS.
3290	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3291	*
3292	* @returns VBox status code.
3293	* @param pVCpu Pointer to the VMCPU.
3294	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3295	* out-of-sync. Make sure to update the required fields
3296	* before using them.
3297	*
3298	* @remarks No-long-jump zone!!!
3299	*/
3300	static int hmR0VmxLoadGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3301	{
3302	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3303	return VINF_SUCCESS;
3304
3305	#ifdef VBOX_STRICT
3306	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3307	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3308	{
3309	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3310	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3311	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3312	}
3313	#endif
3314
3315	int rc = VERR_INTERNAL_ERROR_5;
3316	PVM pVM = pVCpu->CTX_SUFF(pVM);
3317	bool fInterceptDB = false;
3318	bool fInterceptMovDRx = false;
3319	if (DBGFIsStepping(pVCpu))
3320	{
3321	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3322	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3323	{
3324	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3325	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3326	AssertRCReturn(rc, rc);
3327	Assert(fInterceptDB == false);
3328	}
3329	else
3330	{
3331	fInterceptDB = true;
3332	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3333	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RFLAGS;
3334	}
3335	}
3336
3337	if (CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3338	{
3339	if (!CPUMIsHyperDebugStateActive(pVCpu))
3340	{
3341	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3342	AssertRC(rc);
3343	}
3344	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3345	fInterceptMovDRx = true;
3346	}
3347	else if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3348	{
3349	if (!CPUMIsGuestDebugStateActive(pVCpu))
3350	{
3351	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3352	AssertRC(rc);
3353	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3354	}
3355	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3356	Assert(fInterceptMovDRx == false);
3357	}
3358	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3359	{
3360	/* For the first time we would need to intercept MOV DRx accesses even when the guest debug registers aren't loaded. */
3361	fInterceptMovDRx = true;
3362	}
3363
3364	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3365	if (fInterceptDB)
3366	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3367	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3368	{
3369	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3370	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3371	#endif
3372	}
3373
3374	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3375	if (fInterceptMovDRx)
3376	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3377	else
3378	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3379
3380	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3381	AssertRCReturn(rc, rc);
3382	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3383	AssertRCReturn(rc, rc);
3384
3385	/* The guest's view of its DR7 is unblemished. Use 32-bit write as upper 32-bits MBZ as asserted above. */
3386	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
3387	AssertRCReturn(rc, rc);
3388
3389	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3390	return rc;
3391	}
3392
3393
3394	#ifdef VBOX_STRICT
3395	/**
3396	* Strict function to validate segment registers.
3397	*
3398	* @remarks Requires CR0.
3399	*/
3400	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3401	{
3402	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3403	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3404	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3405	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3406	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3407	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3408	{
3409	/* Protected mode checks */
3410	/* CS */
3411	Assert(pCtx->cs.Attr.n.u1Present);
3412	Assert(!(pCtx->cs.Attr.u & 0xf00));
3413	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3414	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3415	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3416	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3417	\|\| (pCtx->cs.Attr.n.u1Granularity));
3418	/* CS cannot be loaded with NULL in protected mode. */
3419	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3420	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3421	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3422	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3423	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3424	else
3425	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3426	/* SS */
3427	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3428	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3429	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0));
3430	if ( !(pCtx->cr0 & X86_CR0_PE)
3431	\|\| pCtx->cs.Attr.n.u4Type == 3)
3432	{
3433	Assert(!pCtx->ss.Attr.n.u2Dpl);
3434	}
3435	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
3436	{
3437	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3438	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3439	Assert(pCtx->ss.Attr.n.u1Present);
3440	Assert(!(pCtx->ss.Attr.u & 0xf00));
3441	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3442	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3443	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3444	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3445	\|\| (pCtx->ss.Attr.n.u1Granularity));
3446	}
3447	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3448	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
3449	{
3450	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3451	Assert(pCtx->ds.Attr.n.u1Present);
3452	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3453	Assert(!(pCtx->ds.Attr.u & 0xf00));
3454	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3455	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3456	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3457	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3458	\|\| (pCtx->ds.Attr.n.u1Granularity));
3459	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3460	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3461	}
3462	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
3463	{
3464	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3465	Assert(pCtx->es.Attr.n.u1Present);
3466	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3467	Assert(!(pCtx->es.Attr.u & 0xf00));
3468	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3469	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3470	\|\| !(pCtx->es.Attr.n.u1Granularity));
3471	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3472	\|\| (pCtx->es.Attr.n.u1Granularity));
3473	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3474	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3475	}
3476	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
3477	{
3478	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3479	Assert(pCtx->fs.Attr.n.u1Present);
3480	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3481	Assert(!(pCtx->fs.Attr.u & 0xf00));
3482	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3483	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3484	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3485	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3486	\|\| (pCtx->fs.Attr.n.u1Granularity));
3487	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3488	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3489	}
3490	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
3491	{
3492	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3493	Assert(pCtx->gs.Attr.n.u1Present);
3494	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3495	Assert(!(pCtx->gs.Attr.u & 0xf00));
3496	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3497	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3498	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3499	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3500	\|\| (pCtx->gs.Attr.n.u1Granularity));
3501	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3502	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3503	}
3504	/* 64-bit capable CPUs. */
3505	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3506	Assert(!(pCtx->cs.u64Base >> 32));
3507	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3508	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3509	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3510	# endif
3511	}
3512	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3513	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3514	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3515	{
3516	/* Real and v86 mode checks. */
3517	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3518	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3519	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3520	{
3521	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3522	}
3523	else
3524	{
3525	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3526	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3527	}
3528
3529	/* CS */
3530	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3531	Assert(pCtx->cs.u32Limit == 0xffff);
3532	Assert(u32CSAttr == 0xf3);
3533	/* SS */
3534	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3535	Assert(pCtx->ss.u32Limit == 0xffff);
3536	Assert(u32SSAttr == 0xf3);
3537	/* DS */
3538	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3539	Assert(pCtx->ds.u32Limit == 0xffff);
3540	Assert(u32DSAttr == 0xf3);
3541	/* ES */
3542	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3543	Assert(pCtx->es.u32Limit == 0xffff);
3544	Assert(u32ESAttr == 0xf3);
3545	/* FS */
3546	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3547	Assert(pCtx->fs.u32Limit == 0xffff);
3548	Assert(u32FSAttr == 0xf3);
3549	/* GS */
3550	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3551	Assert(pCtx->gs.u32Limit == 0xffff);
3552	Assert(u32GSAttr == 0xf3);
3553	/* 64-bit capable CPUs. */
3554	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3555	Assert(!(pCtx->cs.u64Base >> 32));
3556	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3557	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3558	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3559	# endif
3560	}
3561	}
3562	#endif /* VBOX_STRICT */
3563
3564
3565	/**
3566	* Writes a guest segment register into the guest-state area in the VMCS.
3567	*
3568	* @returns VBox status code.
3569	* @param pVCpu Pointer to the VMCPU.
3570	* @param idxSel Index of the selector in the VMCS.
3571	* @param idxLimit Index of the segment limit in the VMCS.
3572	* @param idxBase Index of the segment base in the VMCS.
3573	* @param idxAccess Index of the access rights of the segment in the VMCS.
3574	* @param pSelReg Pointer to the segment selector.
3575	* @param pCtx Pointer to the guest-CPU context.
3576	*
3577	* @remarks No-long-jump zone!!!
3578	*/
3579	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3580	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3581	{
3582	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3583	AssertRCReturn(rc, rc);
3584	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3585	AssertRCReturn(rc, rc);
3586	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3587	AssertRCReturn(rc, rc);
3588
3589	uint32_t u32Access = pSelReg->Attr.u;
3590	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3591	{
3592	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3593	u32Access = 0xf3;
3594	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3595	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3596	}
3597	else
3598	{
3599	/*
3600	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3601	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3602	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3603	* loaded in protected-mode have their attribute as 0.
3604	*/
3605	if (!u32Access)
3606	u32Access = X86DESCATTR_UNUSABLE;
3607	}
3608
3609	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3610	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3611	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3612
3613	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3614	AssertRCReturn(rc, rc);
3615	return rc;
3616	}
3617
3618
3619	/**
3620	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3621	* into the guest-state area in the VMCS.
3622	*
3623	* @returns VBox status code.
3624	* @param pVM Pointer to the VM.
3625	* @param pVCPU Pointer to the VMCPU.
3626	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3627	* out-of-sync. Make sure to update the required fields
3628	* before using them.
3629	*
3630	* @remarks Requires CR0 (strict builds validation).
3631	* @remarks No-long-jump zone!!!
3632	*/
3633	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3634	{
3635	int rc = VERR_INTERNAL_ERROR_5;
3636	PVM pVM = pVCpu->CTX_SUFF(pVM);
3637
3638	/*
3639	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3640	*/
3641	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3642	{
3643	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3644	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3645	{
3646	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pMixedCtx->cs.Attr.u;
3647	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pMixedCtx->ss.Attr.u;
3648	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pMixedCtx->ds.Attr.u;
3649	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pMixedCtx->es.Attr.u;
3650	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pMixedCtx->fs.Attr.u;
3651	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pMixedCtx->gs.Attr.u;
3652	}
3653
3654	#ifdef VBOX_WITH_REM
3655	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3656	{
3657	Assert(pVM->hm.s.vmx.pRealModeTSS);
3658	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3659	if ( pVCpu->hm.s.vmx.fWasInRealMode
3660	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3661	{
3662	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3663	in real-mode (e.g. OpenBSD 4.0) */
3664	REMFlushTBs(pVM);
3665	Log4(("Load: Switch to protected mode detected!\n"));
3666	pVCpu->hm.s.vmx.fWasInRealMode = false;
3667	}
3668	}
3669	#endif
3670	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3671	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3672	AssertRCReturn(rc, rc);
3673	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3674	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3675	AssertRCReturn(rc, rc);
3676	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3677	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3678	AssertRCReturn(rc, rc);
3679	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3680	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3681	AssertRCReturn(rc, rc);
3682	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3683	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3684	AssertRCReturn(rc, rc);
3685	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3686	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3687	AssertRCReturn(rc, rc);
3688
3689	Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
3690	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
3691	#ifdef VBOX_STRICT
3692	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3693	#endif
3694	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3695	}
3696
3697	/*
3698	* Guest TR.
3699	*/
3700	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3701	{
3702	/*
3703	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3704	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3705	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3706	*/
3707	uint16_t u16Sel = 0;
3708	uint32_t u32Limit = 0;
3709	uint64_t u64Base = 0;
3710	uint32_t u32AccessRights = 0;
3711
3712	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3713	{
3714	u16Sel = pMixedCtx->tr.Sel;
3715	u32Limit = pMixedCtx->tr.u32Limit;
3716	u64Base = pMixedCtx->tr.u64Base;
3717	u32AccessRights = pMixedCtx->tr.Attr.u;
3718	}
3719	else
3720	{
3721	Assert(pVM->hm.s.vmx.pRealModeTSS);
3722	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3723
3724	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3725	RTGCPHYS GCPhys;
3726	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3727	AssertRCReturn(rc, rc);
3728
3729	X86DESCATTR DescAttr;
3730	DescAttr.u = 0;
3731	DescAttr.n.u1Present = 1;
3732	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3733
3734	u16Sel = 0;
3735	u32Limit = HM_VTX_TSS_SIZE;
3736	u64Base = GCPhys; /* in real-mode phys = virt. */
3737	u32AccessRights = DescAttr.u;
3738	}
3739
3740	/* Validate. */
3741	Assert(!(u16Sel & RT_BIT(2)));
3742	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3743	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3744	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3745	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3746	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3747	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3748	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3749	Assert( (u32Limit & 0xfff) == 0xfff
3750	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3751	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3752	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3753
3754	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3755	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3756	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3757	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3758
3759	Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3760	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3761	}
3762
3763	/*
3764	* Guest GDTR.
3765	*/
3766	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3767	{
3768	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3769	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3770
3771	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3772	Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3773	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3774	}
3775
3776	/*
3777	* Guest LDTR.
3778	*/
3779	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3780	{
3781	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3782	uint32_t u32Access = 0;
3783	if (!pMixedCtx->ldtr.Attr.u)
3784	u32Access = X86DESCATTR_UNUSABLE;
3785	else
3786	u32Access = pMixedCtx->ldtr.Attr.u;
3787
3788	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3789	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3790	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3791	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3792
3793	/* Validate. */
3794	if (!(u32Access & X86DESCATTR_UNUSABLE))
3795	{
3796	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3797	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3798	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3799	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3800	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3801	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3802	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3803	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3804	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3805	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3806	}
3807
3808	Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3809	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3810	}
3811
3812	/*
3813	* Guest IDTR.
3814	*/
3815	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3816	{
3817	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
3818	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
3819
3820	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3821	Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3822	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3823	}
3824
3825	return VINF_SUCCESS;
3826	}
3827
3828
3829	/**
3830	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3831	* areas. These MSRs will automatically be loaded to the host CPU on every
3832	* successful VM entry and stored from the host CPU on every successful VM exit.
3833	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3834	*
3835	* @returns VBox status code.
3836	* @param pVCpu Pointer to the VMCPU.
3837	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3838	* out-of-sync. Make sure to update the required fields
3839	* before using them.
3840	*
3841	* @remarks No-long-jump zone!!!
3842	*/
3843	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3844	{
3845	AssertPtr(pVCpu);
3846	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3847
3848	/*
3849	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3850	*/
3851	int rc = VINF_SUCCESS;
3852	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3853	{
3854	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3855	PVM pVM = pVCpu->CTX_SUFF(pVM);
3856	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3857	uint32_t cGuestMsrs = 0;
3858
3859	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3860	/** @todo r=ramshankar: Optimize this further to do lazy restoration and only
3861	* when the guest really is in 64-bit mode. */
3862	bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3863	if (fSupportsLongMode)
3864	{
3865	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3866	pGuestMsr->u32Reserved = 0;
3867	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3868	pGuestMsr++; cGuestMsrs++;
3869	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3870	pGuestMsr->u32Reserved = 0;
3871	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3872	pGuestMsr++; cGuestMsrs++;
3873	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3874	pGuestMsr->u32Reserved = 0;
3875	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3876	pGuestMsr++; cGuestMsrs++;
3877	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3878	pGuestMsr->u32Reserved = 0;
3879	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3880	pGuestMsr++; cGuestMsrs++;
3881	}
3882
3883	/*
3884	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3885	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3886	*/
3887	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3888	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3889	{
3890	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3891	pGuestMsr->u32Reserved = 0;
3892	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3893	AssertRCReturn(rc, rc);
3894	pGuestMsr++; cGuestMsrs++;
3895	}
3896
3897	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3898	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3899	{
3900	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3901	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3902	}
3903
3904	/* Update the VCPU's copy of the guest MSR count. */
3905	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3906	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3907	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3908	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
3909
3910	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3911	}
3912
3913	/*
3914	* Guest Sysenter MSRs.
3915	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3916	* VM-exits on WRMSRs for these MSRs.
3917	*/
3918	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3919	{
3920	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
3921	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3922	}
3923	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3924	{
3925	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
3926	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3927	}
3928	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3929	{
3930	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
3931	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3932	}
3933
3934	return rc;
3935	}
3936
3937
3938	/**
3939	* Loads the guest activity state into the guest-state area in the VMCS.
3940	*
3941	* @returns VBox status code.
3942	* @param pVCpu Pointer to the VMCPU.
3943	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3944	* out-of-sync. Make sure to update the required fields
3945	* before using them.
3946	*
3947	* @remarks No-long-jump zone!!!
3948	*/
3949	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
3950	{
3951	/** @todo See if we can make use of other states, e.g.
3952	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3953	int rc = VINF_SUCCESS;
3954	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3955	{
3956	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3957	AssertRCReturn(rc, rc);
3958	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3959	}
3960	return rc;
3961	}
3962
3963
3964	/**
3965	* Sets up the appropriate function to run guest code.
3966	*
3967	* @returns VBox status code.
3968	* @param pVCpu Pointer to the VMCPU.
3969	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3970	* out-of-sync. Make sure to update the required fields
3971	* before using them.
3972	*
3973	* @remarks No-long-jump zone!!!
3974	*/
3975	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3976	{
3977	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3978	{
3979	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3980	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3981	#endif
3982	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3983	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3984	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3985	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3986	#else
3987	/* 64-bit host or hybrid host. */
3988	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3989	#endif
3990	}
3991	else
3992	{
3993	/* Guest is not in long mode, use the 32-bit handler. */
3994	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3995	}
3996	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3997	return VINF_SUCCESS;
3998	}
3999
4000
4001	/**
4002	* Wrapper for running the guest code in VT-x.
4003	*
4004	* @returns VBox strict status code.
4005	* @param pVM Pointer to the VM.
4006	* @param pVCpu Pointer to the VMCPU.
4007	* @param pCtx Pointer to the guest-CPU context.
4008	*
4009	* @remarks No-long-jump zone!!!
4010	*/
4011	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4012	{
4013	/*
4014	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4015	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4016	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4017	*/
4018	#ifdef VBOX_WITH_KERNEL_USING_XMM
4019	return HMR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4020	#else
4021	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4022	#endif
4023	}
4024
4025
4026	/**
4027	* Reports world-switch error and dumps some useful debug info.
4028	*
4029	* @param pVM Pointer to the VM.
4030	* @param pVCpu Pointer to the VMCPU.
4031	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4032	* @param pCtx Pointer to the guest-CPU context.
4033	* @param pVmxTransient Pointer to the VMX transient structure (only
4034	* exitReason updated).
4035	*/
4036	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4037	{
4038	Assert(pVM);
4039	Assert(pVCpu);
4040	Assert(pCtx);
4041	Assert(pVmxTransient);
4042	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4043
4044	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4045	switch (rcVMRun)
4046	{
4047	case VERR_VMX_INVALID_VMXON_PTR:
4048	AssertFailed();
4049	break;
4050	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4051	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4052	{
4053	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4054	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4055	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4056	AssertRC(rc);
4057
4058	#ifdef VBOX_STRICT
4059	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4060	pVmxTransient->uExitReason));
4061	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4062	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4063	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4064	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4065	else
4066	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4067
4068	/* VMX control bits. */
4069	uint32_t u32Val;
4070	uint64_t u64Val;
4071	HMVMXHCUINTREG uHCReg;
4072	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4073	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4074	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4075	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4076	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4077	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4078	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4079	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4080	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4081	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4082	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4083	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4084	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4085	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4086	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4087	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4088	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4089	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4090	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4091	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4092	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4093	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4094	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4095	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4096	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4097	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4098	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4099	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4100	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4101	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4102	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4103	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4104	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4105	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4106	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4107	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4108	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4109	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4110	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4111	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4112	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4113	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4114
4115	/* Guest bits. */
4116	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4117	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4118	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4119	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4120	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4121	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4122	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4123	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4124
4125	/* Host bits. */
4126	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4127	Log4(("Host CR0 %#RHr\n", uHCReg));
4128	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4129	Log4(("Host CR3 %#RHr\n", uHCReg));
4130	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4131	Log4(("Host CR4 %#RHr\n", uHCReg));
4132
4133	RTGDTR HostGdtr;
4134	PCX86DESCHC pDesc;
4135	ASMGetGDTR(&HostGdtr);
4136	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4137	Log4(("Host CS %#08x\n", u32Val));
4138	if (u32Val < HostGdtr.cbGdt)
4139	{
4140	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4141	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4142	}
4143
4144	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4145	Log4(("Host DS %#08x\n", u32Val));
4146	if (u32Val < HostGdtr.cbGdt)
4147	{
4148	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4149	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4150	}
4151
4152	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4153	Log4(("Host ES %#08x\n", u32Val));
4154	if (u32Val < HostGdtr.cbGdt)
4155	{
4156	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4157	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4158	}
4159
4160	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4161	Log4(("Host FS %#08x\n", u32Val));
4162	if (u32Val < HostGdtr.cbGdt)
4163	{
4164	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4165	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4166	}
4167
4168	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4169	Log4(("Host GS %#08x\n", u32Val));
4170	if (u32Val < HostGdtr.cbGdt)
4171	{
4172	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4173	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4174	}
4175
4176	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4177	Log4(("Host SS %#08x\n", u32Val));
4178	if (u32Val < HostGdtr.cbGdt)
4179	{
4180	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4181	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4182	}
4183
4184	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4185	Log4(("Host TR %#08x\n", u32Val));
4186	if (u32Val < HostGdtr.cbGdt)
4187	{
4188	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4189	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4190	}
4191
4192	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4193	Log4(("Host TR Base %#RHv\n", uHCReg));
4194	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4195	Log4(("Host GDTR Base %#RHv\n", uHCReg));
4196	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4197	Log4(("Host IDTR Base %#RHv\n", uHCReg));
4198	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4199	Log4(("Host SYSENTER CS %#08x\n", u32Val));
4200	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4201	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4202	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4203	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4204	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4205	Log4(("Host RSP %#RHv\n", uHCReg));
4206	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4207	Log4(("Host RIP %#RHv\n", uHCReg));
4208	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4209	if (HMVMX_IS_64BIT_HOST_MODE())
4210	{
4211	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4212	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4213	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4214	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4215	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4216	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4217	}
4218	# endif
4219	#endif /* VBOX_STRICT */
4220	break;
4221	}
4222
4223	default:
4224	/* Impossible */
4225	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4226	break;
4227	}
4228	NOREF(pVM);
4229	}
4230
4231
4232	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4233	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4234	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4235	#endif
4236	#ifdef VBOX_STRICT
4237	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4238	{
4239	switch (idxField)
4240	{
4241	case VMX_VMCS_GUEST_RIP:
4242	case VMX_VMCS_GUEST_RSP:
4243	case VMX_VMCS_GUEST_SYSENTER_EIP:
4244	case VMX_VMCS_GUEST_SYSENTER_ESP:
4245	case VMX_VMCS_GUEST_GDTR_BASE:
4246	case VMX_VMCS_GUEST_IDTR_BASE:
4247	case VMX_VMCS_GUEST_CS_BASE:
4248	case VMX_VMCS_GUEST_DS_BASE:
4249	case VMX_VMCS_GUEST_ES_BASE:
4250	case VMX_VMCS_GUEST_FS_BASE:
4251	case VMX_VMCS_GUEST_GS_BASE:
4252	case VMX_VMCS_GUEST_SS_BASE:
4253	case VMX_VMCS_GUEST_LDTR_BASE:
4254	case VMX_VMCS_GUEST_TR_BASE:
4255	case VMX_VMCS_GUEST_CR3:
4256	return true;
4257	}
4258	return false;
4259	}
4260
4261	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4262	{
4263	switch (idxField)
4264	{
4265	/* Read-only fields. */
4266	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4267	return true;
4268	}
4269	/* Remaining readable fields should also be writable. */
4270	return hmR0VmxIsValidWriteField(idxField);
4271	}
4272	#endif /* VBOX_STRICT */
4273
4274
4275	/**
4276	* Executes the specified handler in 64-bit mode.
4277	*
4278	* @returns VBox status code.
4279	* @param pVM Pointer to the VM.
4280	* @param pVCpu Pointer to the VMCPU.
4281	* @param pCtx Pointer to the guest CPU context.
4282	* @param enmOp The operation to perform.
4283	* @param cbParam Number of parameters.
4284	* @param paParam Array of 32-bit parameters.
4285	*/
4286	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4287	uint32_t *paParam)
4288	{
4289	int rc, rc2;
4290	PHMGLOBLCPUINFO pCpu;
4291	RTHCPHYS HCPhysCpuPage;
4292	RTCCUINTREG uOldEFlags;
4293
4294	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4295	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4296	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4297	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4298
4299	#ifdef VBOX_STRICT
4300	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4301	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4302
4303	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4304	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4305	#endif
4306
4307	/* Disable interrupts. */
4308	uOldEFlags = ASMIntDisableFlags();
4309
4310	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4311	RTCPUID idHostCpu = RTMpCpuId();
4312	CPUMR0SetLApic(pVM, idHostCpu);
4313	#endif
4314
4315	pCpu = HMR0GetCurrentCpu();
4316	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4317
4318	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4319	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4320
4321	/* Leave VMX Root Mode. */
4322	VMXDisable();
4323
4324	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4325
4326	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4327	CPUMSetHyperEIP(pVCpu, enmOp);
4328	for (int i = (int)cbParam - 1; i >= 0; i--)
4329	CPUMPushHyper(pVCpu, paParam[i]);
4330
4331	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4332
4333	/* Call the switcher. */
4334	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4335	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4336
4337	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4338	/* Make sure the VMX instructions don't cause #UD faults. */
4339	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4340
4341	/* Re-enter VMX Root Mode */
4342	rc2 = VMXEnable(HCPhysCpuPage);
4343	if (RT_FAILURE(rc2))
4344	{
4345	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4346	ASMSetFlags(uOldEFlags);
4347	return rc2;
4348	}
4349
4350	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4351	AssertRC(rc2);
4352	Assert(!(ASMGetFlags() & X86_EFL_IF));
4353	ASMSetFlags(uOldEFlags);
4354	return rc;
4355	}
4356
4357
4358	/**
4359	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4360	* supporting 64-bit guests.
4361	*
4362	* @returns VBox status code.
4363	* @param fResume Whether to VMLAUNCH or VMRESUME.
4364	* @param pCtx Pointer to the guest-CPU context.
4365	* @param pCache Pointer to the VMCS cache.
4366	* @param pVM Pointer to the VM.
4367	* @param pVCpu Pointer to the VMCPU.
4368	*/
4369	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4370	{
4371	uint32_t aParam[6];
4372	PHMGLOBLCPUINFO pCpu = NULL;
4373	RTHCPHYS HCPhysCpuPage = 0;
4374	int rc = VERR_INTERNAL_ERROR_5;
4375
4376	pCpu = HMR0GetCurrentCpu();
4377	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4378
4379	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4380	pCache->uPos = 1;
4381	pCache->interPD = PGMGetInterPaeCR3(pVM);
4382	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4383	#endif
4384
4385	#ifdef VBOX_STRICT
4386	pCache->TestIn.HCPhysCpuPage = 0;
4387	pCache->TestIn.HCPhysVmcs = 0;
4388	pCache->TestIn.pCache = 0;
4389	pCache->TestOut.HCPhysVmcs = 0;
4390	pCache->TestOut.pCache = 0;
4391	pCache->TestOut.pCtx = 0;
4392	pCache->TestOut.eflags = 0;
4393	#endif
4394
4395	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4396	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4397	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4398	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4399	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4400	aParam[5] = 0;
4401
4402	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4403	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4404	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4405	#endif
4406	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4407
4408	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4409	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4410	Assert(pCtx->dr[4] == 10);
4411	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4412	#endif
4413
4414	#ifdef VBOX_STRICT
4415	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4416	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4417	pVCpu->hm.s.vmx.HCPhysVmcs));
4418	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4419	pCache->TestOut.HCPhysVmcs));
4420	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4421	pCache->TestOut.pCache));
4422	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4423	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4424	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4425	pCache->TestOut.pCtx));
4426	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4427	#endif
4428	return rc;
4429	}
4430
4431
4432	/**
4433	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4434	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4435	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4436	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4437	*
4438	* @returns VBox status code.
4439	* @param pVM Pointer to the VM.
4440	* @param pVCpu Pointer to the VMCPU.
4441	*/
4442	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4443	{
4444	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4445	{ \
4446	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4447	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4448	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4449	++cReadFields; \
4450	}
4451
4452	AssertPtr(pVM);
4453	AssertPtr(pVCpu);
4454	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4455	uint32_t cReadFields = 0;
4456
4457	/*
4458	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
4459	* and serve to indicate exceptions to the rules.
4460	*/
4461
4462	/* Guest-natural selector base fields. */
4463	#if 0
4464	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4465	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4466	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4467	#endif
4468	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4469	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4470	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4471	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4472	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4473	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4474	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4475	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4476	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4477	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4478	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4479	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4480	#if 0
4481	/* Unused natural width guest-state fields. */
4482	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4483	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4484	#endif
4485	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4486	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4487
4488	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4489	#if 0
4490	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4491	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4492	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4493	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4494	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4495	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4496	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4497	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4498	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4499	#endif
4500
4501	/* Natural width guest-state fields. */
4502	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4503	#if 0
4504	/* Currently unused field. */
4505	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4506	#endif
4507
4508	if (pVM->hm.s.fNestedPaging)
4509	{
4510	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4511	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4512	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4513	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4514	}
4515	else
4516	{
4517	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4518	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4519	}
4520
4521	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4522	return VINF_SUCCESS;
4523	}
4524
4525
4526	/**
4527	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4528	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4529	* darwin, running 64-bit guests).
4530	*
4531	* @returns VBox status code.
4532	* @param pVCpu Pointer to the VMCPU.
4533	* @param idxField The VMCS field encoding.
4534	* @param u64Val 16, 32 or 64 bits value.
4535	*/
4536	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4537	{
4538	int rc;
4539	switch (idxField)
4540	{
4541	/*
4542	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4543	*/
4544	/* 64-bit Control fields. */
4545	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4546	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4547	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4548	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4549	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4550	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4551	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4552	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4553	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4554	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4555	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4556	case VMX_VMCS64_CTRL_EPTP_FULL:
4557	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4558	/* 64-bit Guest-state fields. */
4559	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4560	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4561	case VMX_VMCS64_GUEST_PAT_FULL:
4562	case VMX_VMCS64_GUEST_EFER_FULL:
4563	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4564	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4565	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4566	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4567	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4568	/* 64-bit Host-state fields. */
4569	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4570	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4571	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4572	{
4573	rc = VMXWriteVmcs32(idxField, u64Val);
4574	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4575	break;
4576	}
4577
4578	/*
4579	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4580	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4581	*/
4582	/* Natural-width Guest-state fields. */
4583	case VMX_VMCS_GUEST_CR3:
4584	case VMX_VMCS_GUEST_ES_BASE:
4585	case VMX_VMCS_GUEST_CS_BASE:
4586	case VMX_VMCS_GUEST_SS_BASE:
4587	case VMX_VMCS_GUEST_DS_BASE:
4588	case VMX_VMCS_GUEST_FS_BASE:
4589	case VMX_VMCS_GUEST_GS_BASE:
4590	case VMX_VMCS_GUEST_LDTR_BASE:
4591	case VMX_VMCS_GUEST_TR_BASE:
4592	case VMX_VMCS_GUEST_GDTR_BASE:
4593	case VMX_VMCS_GUEST_IDTR_BASE:
4594	case VMX_VMCS_GUEST_RSP:
4595	case VMX_VMCS_GUEST_RIP:
4596	case VMX_VMCS_GUEST_SYSENTER_ESP:
4597	case VMX_VMCS_GUEST_SYSENTER_EIP:
4598	{
4599	if (!(u64Val >> 32))
4600	{
4601	/* If this field is 64-bit, VT-x will zero out the top bits. */
4602	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4603	}
4604	else
4605	{
4606	/* Assert that only the 32->64 switcher case should ever come here. */
4607	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4608	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4609	}
4610	break;
4611	}
4612
4613	default:
4614	{
4615	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4616	rc = VERR_INVALID_PARAMETER;
4617	break;
4618	}
4619	}
4620	AssertRCReturn(rc, rc);
4621	return rc;
4622	}
4623
4624
4625	/**
4626	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4627	* hosts (except darwin) for 64-bit guests.
4628	*
4629	* @param pVCpu Pointer to the VMCPU.
4630	* @param idxField The VMCS field encoding.
4631	* @param u64Val 16, 32 or 64 bits value.
4632	*/
4633	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4634	{
4635	AssertPtr(pVCpu);
4636	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4637
4638	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4639	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4640
4641	/* Make sure there are no duplicates. */
4642	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4643	{
4644	if (pCache->Write.aField[i] == idxField)
4645	{
4646	pCache->Write.aFieldVal[i] = u64Val;
4647	return VINF_SUCCESS;
4648	}
4649	}
4650
4651	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4652	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4653	pCache->Write.cValidEntries++;
4654	return VINF_SUCCESS;
4655	}
4656
4657	/* Enable later when the assembly code uses these as callbacks. */
4658	#if 0
4659	/*
4660	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4661	*
4662	* @param pVCpu Pointer to the VMCPU.
4663	* @param pCache Pointer to the VMCS cache.
4664	*
4665	* @remarks No-long-jump zone!!!
4666	*/
4667	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4668	{
4669	AssertPtr(pCache);
4670	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4671	{
4672	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4673	AssertRC(rc);
4674	}
4675	pCache->Write.cValidEntries = 0;
4676	}
4677
4678
4679	/**
4680	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4681	*
4682	* @param pVCpu Pointer to the VMCPU.
4683	* @param pCache Pointer to the VMCS cache.
4684	*
4685	* @remarks No-long-jump zone!!!
4686	*/
4687	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4688	{
4689	AssertPtr(pCache);
4690	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4691	{
4692	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4693	AssertRC(rc);
4694	}
4695	}
4696	#endif
4697	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4698
4699
4700	/**
4701	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4702	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4703	* timer.
4704	*
4705	* @returns VBox status code.
4706	* @param pVCpu Pointer to the VMCPU.
4707	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4708	* out-of-sync. Make sure to update the required fields
4709	* before using them.
4710	* @remarks No-long-jump zone!!!
4711	*/
4712	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4713	{
4714	int rc = VERR_INTERNAL_ERROR_5;
4715	bool fOffsettedTsc = false;
4716	PVM pVM = pVCpu->CTX_SUFF(pVM);
4717	if (pVM->hm.s.vmx.fUsePreemptTimer)
4718	{
4719	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4720
4721	/* Make sure the returned values have sane upper and lower boundaries. */
4722	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4723	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4724	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4725	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4726
4727	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4728	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4729	}
4730	else
4731	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4732
4733	if (fOffsettedTsc)
4734	{
4735	uint64_t u64CurTSC = ASMReadTSC();
4736	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4737	{
4738	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4739	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4740
4741	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4742	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4743	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4744	}
4745	else
4746	{
4747	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4748	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4749	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4750	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4751	}
4752	}
4753	else
4754	{
4755	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4756	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4757	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4758	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4759	}
4760	}
4761
4762
4763	/**
4764	* Determines if an exception is a contributory exception. Contributory
4765	* exceptions are ones which can cause double-faults. Page-fault is
4766	* intentionally not included here as it's a conditional contributory exception.
4767	*
4768	* @returns true if the exception is contributory, false otherwise.
4769	* @param uVector The exception vector.
4770	*/
4771	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4772	{
4773	switch (uVector)
4774	{
4775	case X86_XCPT_GP:
4776	case X86_XCPT_SS:
4777	case X86_XCPT_NP:
4778	case X86_XCPT_TS:
4779	case X86_XCPT_DE:
4780	return true;
4781	default:
4782	break;
4783	}
4784	return false;
4785	}
4786
4787
4788	/**
4789	* Sets an event as a pending event to be injected into the guest.
4790	*
4791	* @param pVCpu Pointer to the VMCPU.
4792	* @param u32IntrInfo The VM-entry interruption-information field.
4793	* @param cbInstr The VM-entry instruction length in bytes (for software
4794	* interrupts, exceptions and privileged software
4795	* exceptions).
4796	* @param u32ErrCode The VM-entry exception error code.
4797	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4798	* page-fault.
4799	*
4800	* @remarks Statistics counter assumes this is a guest event being injected or
4801	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
4802	* always incremented.
4803	*/
4804	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4805	RTGCUINTPTR GCPtrFaultAddress)
4806	{
4807	Assert(!pVCpu->hm.s.Event.fPending);
4808	pVCpu->hm.s.Event.fPending = true;
4809	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4810	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4811	pVCpu->hm.s.Event.cbInstr = cbInstr;
4812	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4813
4814	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
4815	}
4816
4817
4818	/**
4819	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4820	*
4821	* @param pVCpu Pointer to the VMCPU.
4822	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4823	* out-of-sync. Make sure to update the required fields
4824	* before using them.
4825	*/
4826	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4827	{
4828	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
4829	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4830	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4831	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4832	}
4833
4834
4835	/**
4836	* Handle a condition that occurred while delivering an event through the guest
4837	* IDT.
4838	*
4839	* @returns VBox status code (informational error codes included).
4840	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4841	* @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
4842	* continue execution of the guest which will delivery the #DF.
4843	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4844	*
4845	* @param pVCpu Pointer to the VMCPU.
4846	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4847	* out-of-sync. Make sure to update the required fields
4848	* before using them.
4849	* @param pVmxTransient Pointer to the VMX transient structure.
4850	*
4851	* @remarks No-long-jump zone!!!
4852	*/
4853	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4854	{
4855	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4856	AssertRC(rc);
4857	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4858	{
4859	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4860	AssertRCReturn(rc, rc);
4861
4862	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4863	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4864	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4865
4866	typedef enum
4867	{
4868	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4869	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4870	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4871	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4872	} VMXREFLECTXCPT;
4873
4874	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4875	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4876	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo))
4877	{
4878	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4879	{
4880	enmReflect = VMXREFLECTXCPT_XCPT;
4881	#ifdef VBOX_STRICT
4882	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4883	&& uExitVector == X86_XCPT_PF)
4884	{
4885	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
4886	}
4887	#endif
4888	if ( uExitVector == X86_XCPT_PF
4889	&& uIdtVector == X86_XCPT_PF)
4890	{
4891	pVmxTransient->fVectoringPF = true;
4892	Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
4893	}
4894	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4895	&& hmR0VmxIsContributoryXcpt(uExitVector)
4896	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
4897	\|\| uIdtVector == X86_XCPT_PF))
4898	{
4899	enmReflect = VMXREFLECTXCPT_DF;
4900	}
4901	else if (uIdtVector == X86_XCPT_DF)
4902	enmReflect = VMXREFLECTXCPT_TF;
4903	}
4904	else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
4905	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
4906	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
4907	{
4908	/*
4909	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4910	* (whatever they are) as they reoccur when restarting the instruction.
4911	*/
4912	enmReflect = VMXREFLECTXCPT_XCPT;
4913	}
4914	}
4915	else
4916	{
4917	/*
4918	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
4919	* interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
4920	* original exception to the guest after handling the VM-exit.
4921	*/
4922	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
4923	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
4924	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
4925	{
4926	enmReflect = VMXREFLECTXCPT_XCPT;
4927	}
4928	}
4929
4930	switch (enmReflect)
4931	{
4932	case VMXREFLECTXCPT_XCPT:
4933	{
4934	Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4935	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4936	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
4937
4938	uint32_t u32ErrCode = 0;
4939	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
4940	{
4941	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4942	AssertRCReturn(rc, rc);
4943	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4944	}
4945
4946	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
4947	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
4948	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
4949	rc = VINF_SUCCESS;
4950	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
4951	pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
4952
4953	break;
4954	}
4955
4956	case VMXREFLECTXCPT_DF:
4957	{
4958	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
4959	rc = VINF_HM_DOUBLE_FAULT;
4960	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
4961	pVCpu->hm.s.Event.u64IntrInfo, uIdtVector, uExitVector));
4962
4963	break;
4964	}
4965
4966	case VMXREFLECTXCPT_TF:
4967	{
4968	rc = VINF_EM_RESET;
4969	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
4970	uExitVector));
4971	break;
4972	}
4973
4974	default:
4975	Assert(rc == VINF_SUCCESS);
4976	break;
4977	}
4978	}
4979	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
4980	return rc;
4981	}
4982
4983
4984	/**
4985	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4986	*
4987	* @returns VBox status code.
4988	* @param pVCpu Pointer to the VMCPU.
4989	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4990	* out-of-sync. Make sure to update the required fields
4991	* before using them.
4992	*
4993	* @remarks No-long-jump zone!!!
4994	*/
4995	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4996	{
4997	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
4998	{
4999	uint32_t uVal = 0;
5000	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5001	AssertRCReturn(rc, rc);
5002	uint32_t uShadow = 0;
5003	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5004	AssertRCReturn(rc, rc);
5005
5006	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5007	CPUMSetGuestCR0(pVCpu, uVal);
5008	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
5009	}
5010	return VINF_SUCCESS;
5011	}
5012
5013
5014	/**
5015	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5016	*
5017	* @returns VBox status code.
5018	* @param pVCpu Pointer to the VMCPU.
5019	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5020	* out-of-sync. Make sure to update the required fields
5021	* before using them.
5022	*
5023	* @remarks No-long-jump zone!!!
5024	*/
5025	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5026	{
5027	int rc = VINF_SUCCESS;
5028	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
5029	{
5030	uint32_t uVal = 0;
5031	uint32_t uShadow = 0;
5032	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5033	AssertRCReturn(rc, rc);
5034	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5035	AssertRCReturn(rc, rc);
5036
5037	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5038	CPUMSetGuestCR4(pVCpu, uVal);
5039	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
5040	}
5041	return rc;
5042	}
5043
5044
5045	/**
5046	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
5047	*
5048	* @returns VBox status code.
5049	* @param pVCpu Pointer to the VMCPU.
5050	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5051	* out-of-sync. Make sure to update the required fields
5052	* before using them.
5053	*
5054	* @remarks No-long-jump zone!!!
5055	*/
5056	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5057	{
5058	int rc = VINF_SUCCESS;
5059	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5060	{
5061	uint64_t u64Val = 0;
5062	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5063	AssertRCReturn(rc, rc);
5064
5065	pMixedCtx->rip = u64Val;
5066	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
5067	}
5068	return rc;
5069	}
5070
5071
5072	/**
5073	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
5074	*
5075	* @returns VBox status code.
5076	* @param pVCpu Pointer to the VMCPU.
5077	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5078	* out-of-sync. Make sure to update the required fields
5079	* before using them.
5080	*
5081	* @remarks No-long-jump zone!!!
5082	*/
5083	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5084	{
5085	int rc = VINF_SUCCESS;
5086	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5087	{
5088	uint64_t u64Val = 0;
5089	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5090	AssertRCReturn(rc, rc);
5091
5092	pMixedCtx->rsp = u64Val;
5093	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
5094	}
5095	return rc;
5096	}
5097
5098
5099	/**
5100	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5101	*
5102	* @returns VBox status code.
5103	* @param pVCpu Pointer to the VMCPU.
5104	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5105	* out-of-sync. Make sure to update the required fields
5106	* before using them.
5107	*
5108	* @remarks No-long-jump zone!!!
5109	*/
5110	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5111	{
5112	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5113	{
5114	uint32_t uVal = 0;
5115	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5116	AssertRCReturn(rc, rc);
5117
5118	pMixedCtx->eflags.u32 = uVal;
5119	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5120	{
5121	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5122	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5123
5124	pMixedCtx->eflags.Bits.u1VM = 0;
5125	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
5126	}
5127
5128	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
5129	}
5130	return VINF_SUCCESS;
5131	}
5132
5133
5134	/**
5135	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5136	* guest-CPU context.
5137	*/
5138	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5139	{
5140	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5141	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5142	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5143	return rc;
5144	}
5145
5146
5147	/**
5148	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5149	* from the guest-state area in the VMCS.
5150	*
5151	* @param pVCpu Pointer to the VMCPU.
5152	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5153	* out-of-sync. Make sure to update the required fields
5154	* before using them.
5155	*
5156	* @remarks No-long-jump zone!!!
5157	*/
5158	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5159	{
5160	uint32_t uIntrState = 0;
5161	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5162	AssertRC(rc);
5163
5164	if (!uIntrState)
5165	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5166	else
5167	{
5168	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5169	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5170	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5171	AssertRC(rc);
5172	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5173	AssertRC(rc);
5174
5175	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5176	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5177	}
5178	}
5179
5180
5181	/**
5182	* Saves the guest's activity state.
5183	*
5184	* @returns VBox status code.
5185	* @param pVCpu Pointer to the VMCPU.
5186	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5187	* out-of-sync. Make sure to update the required fields
5188	* before using them.
5189	*
5190	* @remarks No-long-jump zone!!!
5191	*/
5192	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5193	{
5194	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5195	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5196	return VINF_SUCCESS;
5197	}
5198
5199
5200	/**
5201	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5202	* the current VMCS into the guest-CPU context.
5203	*
5204	* @returns VBox status code.
5205	* @param pVCpu Pointer to the VMCPU.
5206	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5207	* out-of-sync. Make sure to update the required fields
5208	* before using them.
5209	*
5210	* @remarks No-long-jump zone!!!
5211	*/
5212	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5213	{
5214	int rc = VINF_SUCCESS;
5215	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5216	{
5217	uint32_t u32Val = 0;
5218	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5219	pMixedCtx->SysEnter.cs = u32Val;
5220	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5221	}
5222
5223	uint64_t u64Val = 0;
5224	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5225	{
5226	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5227	pMixedCtx->SysEnter.eip = u64Val;
5228	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5229	}
5230	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5231	{
5232	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5233	pMixedCtx->SysEnter.esp = u64Val;
5234	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5235	}
5236	return rc;
5237	}
5238
5239
5240	/**
5241	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5242	* context.
5243	*
5244	* @returns VBox status code.
5245	* @param pVCpu Pointer to the VMCPU.
5246	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5247	* out-of-sync. Make sure to update the required fields
5248	* before using them.
5249	*
5250	* @remarks No-long-jump zone!!!
5251	*/
5252	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5253	{
5254	int rc = VINF_SUCCESS;
5255	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5256	{
5257	uint64_t u64Val = 0;
5258	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5259	pMixedCtx->fs.u64Base = u64Val;
5260	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5261	}
5262	return rc;
5263	}
5264
5265
5266	/**
5267	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5268	* context.
5269	*
5270	* @returns VBox status code.
5271	* @param pVCpu Pointer to the VMCPU.
5272	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5273	* out-of-sync. Make sure to update the required fields
5274	* before using them.
5275	*
5276	* @remarks No-long-jump zone!!!
5277	*/
5278	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5279	{
5280	int rc = VINF_SUCCESS;
5281	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5282	{
5283	uint64_t u64Val = 0;
5284	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5285	pMixedCtx->gs.u64Base = u64Val;
5286	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5287	}
5288	return rc;
5289	}
5290
5291
5292	/**
5293	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5294	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5295	* and TSC_AUX.
5296	*
5297	* @returns VBox status code.
5298	* @param pVCpu Pointer to the VMCPU.
5299	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5300	* out-of-sync. Make sure to update the required fields
5301	* before using them.
5302	*
5303	* @remarks No-long-jump zone!!!
5304	*/
5305	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5306	{
5307	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5308	return VINF_SUCCESS;
5309
5310	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5311	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5312	{
5313	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5314	pMsr += i;
5315	switch (pMsr->u32IndexMSR)
5316	{
5317	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5318	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5319	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5320	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5321	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5322	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5323	default:
5324	{
5325	AssertFailed();
5326	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5327	}
5328	}
5329	}
5330	#endif
5331
5332	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5333	return VINF_SUCCESS;
5334	}
5335
5336
5337	/**
5338	* Saves the guest control registers from the current VMCS into the guest-CPU
5339	* context.
5340	*
5341	* @returns VBox status code.
5342	* @param pVCpu Pointer to the VMCPU.
5343	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5344	* out-of-sync. Make sure to update the required fields
5345	* before using them.
5346	*
5347	* @remarks No-long-jump zone!!!
5348	*/
5349	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5350	{
5351	/* Guest CR0. Guest FPU. */
5352	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5353	AssertRCReturn(rc, rc);
5354
5355	/* Guest CR4. */
5356	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5357	AssertRCReturn(rc, rc);
5358
5359	/* Guest CR2 - updated always during the world-switch or in #PF. */
5360	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5361	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5362	{
5363	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5364	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5365
5366	PVM pVM = pVCpu->CTX_SUFF(pVM);
5367	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5368	\|\| ( pVM->hm.s.fNestedPaging
5369	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5370	{
5371	uint64_t u64Val = 0;
5372	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5373	if (pMixedCtx->cr3 != u64Val)
5374	{
5375	CPUMSetGuestCR3(pVCpu, u64Val);
5376	if (VMMRZCallRing3IsEnabled(pVCpu))
5377	{
5378	PGMUpdateCR3(pVCpu, u64Val);
5379	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5380	}
5381	else
5382	{
5383	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5384	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5385	}
5386	}
5387
5388	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5389	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5390	{
5391	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5392	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5393	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5394	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5395
5396	if (VMMRZCallRing3IsEnabled(pVCpu))
5397	{
5398	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5399	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5400	}
5401	else
5402	{
5403	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5404	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5405	}
5406	}
5407	}
5408
5409	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5410	}
5411
5412	/*
5413	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5414	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5415	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5416	*
5417	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5418	*/
5419	if (VMMRZCallRing3IsEnabled(pVCpu))
5420	{
5421	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5422	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5423
5424	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5425	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5426
5427	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5428	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5429	}
5430
5431	return rc;
5432	}
5433
5434
5435	/**
5436	* Reads a guest segment register from the current VMCS into the guest-CPU
5437	* context.
5438	*
5439	* @returns VBox status code.
5440	* @param pVCpu Pointer to the VMCPU.
5441	* @param idxSel Index of the selector in the VMCS.
5442	* @param idxLimit Index of the segment limit in the VMCS.
5443	* @param idxBase Index of the segment base in the VMCS.
5444	* @param idxAccess Index of the access rights of the segment in the VMCS.
5445	* @param pSelReg Pointer to the segment selector.
5446	*
5447	* @remarks No-long-jump zone!!!
5448	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5449	* macro as that takes care of whether to read from the VMCS cache or
5450	* not.
5451	*/
5452	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5453	PCPUMSELREG pSelReg)
5454	{
5455	uint32_t u32Val = 0;
5456	int rc = VMXReadVmcs32(idxSel, &u32Val);
5457	AssertRCReturn(rc, rc);
5458	pSelReg->Sel = (uint16_t)u32Val;
5459	pSelReg->ValidSel = (uint16_t)u32Val;
5460	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5461
5462	rc = VMXReadVmcs32(idxLimit, &u32Val);
5463	AssertRCReturn(rc, rc);
5464	pSelReg->u32Limit = u32Val;
5465
5466	uint64_t u64Val = 0;
5467	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5468	AssertRCReturn(rc, rc);
5469	pSelReg->u64Base = u64Val;
5470
5471	rc = VMXReadVmcs32(idxAccess, &u32Val);
5472	AssertRCReturn(rc, rc);
5473	pSelReg->Attr.u = u32Val;
5474
5475	/*
5476	* If VT-x marks the segment as unusable, the rest of the attributes are undefined with certain exceptions (some bits in
5477	* CS, SS). Regardless, we have to clear the bits here and only retain the unusable bit because the unusable bit is specific
5478	* to VT-x, everyone else relies on the attribute being zero and have no clue what the unusable bit is.
5479	*
5480	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5481	*
5482	* bird: This isn't quite as simple. VT-x and VBox(!) requires the DPL for SS to be the same as CPL. In 64-bit mode it
5483	* is possible (int/trap/xxx injects does this when switching rings) to load SS with a NULL selector and RPL=CPL.
5484	* The Attr.u = X86DESCATTR_UNUSABLE works fine as long as nobody uses ring-1 or ring-2. VT-x updates the DPL
5485	* correctly in the attributes of SS even when the unusable bit is set, we need to preserve the DPL or we get invalid
5486	* guest state trouble. Try bs2-cpu-hidden-regs-1.
5487	*/
5488	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
5489	{
5490	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5491	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x\n", idxSel, pSelReg->Attr.u));
5492
5493	if (idxSel == VMX_VMCS16_GUEST_FIELD_SS)
5494	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_DPL;
5495	else if (idxSel == VMX_VMCS16_GUEST_FIELD_CS)
5496	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G;
5497	else
5498	pSelReg->Attr.u = X86DESCATTR_UNUSABLE;
5499	}
5500	return VINF_SUCCESS;
5501	}
5502
5503
5504	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5505	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5506	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5507	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5508	#else
5509	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5510	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5511	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5512	#endif
5513
5514
5515	/**
5516	* Saves the guest segment registers from the current VMCS into the guest-CPU
5517	* context.
5518	*
5519	* @returns VBox status code.
5520	* @param pVCpu Pointer to the VMCPU.
5521	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5522	* out-of-sync. Make sure to update the required fields
5523	* before using them.
5524	*
5525	* @remarks No-long-jump zone!!!
5526	*/
5527	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5528	{
5529	/* Guest segment registers. */
5530	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5531	{
5532	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5533	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5534	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5535	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5536	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5537	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5538	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5539
5540	/* Restore segment attributes for real-on-v86 mode hack. */
5541	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5542	{
5543	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5544	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5545	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5546	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5547	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5548	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5549	}
5550	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5551	}
5552
5553	return VINF_SUCCESS;
5554	}
5555
5556
5557	/**
5558	* Saves the guest descriptor table registers and task register from the current
5559	* VMCS into the guest-CPU context.
5560	*
5561	* @returns VBox status code.
5562	* @param pVCpu Pointer to the VMCPU.
5563	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5564	* out-of-sync. Make sure to update the required fields
5565	* before using them.
5566	*
5567	* @remarks No-long-jump zone!!!
5568	*/
5569	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5570	{
5571	int rc = VINF_SUCCESS;
5572
5573	/* Guest LDTR. */
5574	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5575	{
5576	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5577	AssertRCReturn(rc, rc);
5578	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5579	}
5580
5581	/* Guest GDTR. */
5582	uint64_t u64Val = 0;
5583	uint32_t u32Val = 0;
5584	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5585	{
5586	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5587	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5588	pMixedCtx->gdtr.pGdt = u64Val;
5589	pMixedCtx->gdtr.cbGdt = u32Val;
5590	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5591	}
5592
5593	/* Guest IDTR. */
5594	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5595	{
5596	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5597	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5598	pMixedCtx->idtr.pIdt = u64Val;
5599	pMixedCtx->idtr.cbIdt = u32Val;
5600	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5601	}
5602
5603	/* Guest TR. */
5604	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5605	{
5606	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5607	AssertRCReturn(rc, rc);
5608
5609	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5610	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5611	{
5612	rc = VMXLOCAL_READ_SEG(TR, tr);
5613	AssertRCReturn(rc, rc);
5614	}
5615	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5616	}
5617	return rc;
5618	}
5619
5620	#undef VMXLOCAL_READ_SEG
5621
5622
5623	/**
5624	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
5625	* context.
5626	*
5627	* @returns VBox status code.
5628	* @param pVCpu Pointer to the VMCPU.
5629	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5630	* out-of-sync. Make sure to update the required fields
5631	* before using them.
5632	*
5633	* @remarks No-long-jump zone!!!
5634	*/
5635	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5636	{
5637	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5638	{
5639	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
5640	uint32_t u32Val;
5641	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
5642	pMixedCtx->dr[7] = u32Val;
5643
5644	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5645	}
5646	return VINF_SUCCESS;
5647	}
5648
5649
5650	/**
5651	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
5652	*
5653	* @returns VBox status code.
5654	* @param pVCpu Pointer to the VMCPU.
5655	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5656	* out-of-sync. Make sure to update the required fields
5657	* before using them.
5658	*
5659	* @remarks No-long-jump zone!!!
5660	*/
5661	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5662	{
5663	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5664	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5665	return VINF_SUCCESS;
5666	}
5667
5668
5669	/**
5670	* Saves the entire guest state from the currently active VMCS into the
5671	* guest-CPU context. This essentially VMREADs all guest-data.
5672	*
5673	* @returns VBox status code.
5674	* @param pVCpu Pointer to the VMCPU.
5675	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5676	* out-of-sync. Make sure to update the required fields
5677	* before using them.
5678	*/
5679	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5680	{
5681	Assert(pVCpu);
5682	Assert(pMixedCtx);
5683
5684	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5685	return VINF_SUCCESS;
5686
5687	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled again on the ring-3 callback path,
5688	there is no real need to. */
5689	if (VMMRZCallRing3IsEnabled(pVCpu))
5690	VMMR0LogFlushDisable(pVCpu);
5691	else
5692	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5693	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
5694
5695	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5696	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5697
5698	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5699	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5700
5701	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5702	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5703
5704	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5705	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5706
5707	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
5708	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5709
5710	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5711	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5712
5713	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5714	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5715
5716	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5717	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5718
5719	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5720	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5721
5722	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5723	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5724
5725	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5726	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5727
5728	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5729	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5730
5731	if (VMMRZCallRing3IsEnabled(pVCpu))
5732	VMMR0LogFlushEnable(pVCpu);
5733
5734	return rc;
5735	}
5736
5737
5738	/**
5739	* Check per-VM and per-VCPU force flag actions that require us to go back to
5740	* ring-3 for one reason or another.
5741	*
5742	* @returns VBox status code (information status code included).
5743	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5744	* ring-3.
5745	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5746	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5747	* interrupts)
5748	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5749	* all EMTs to be in ring-3.
5750	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5751	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5752	* to the EM loop.
5753	*
5754	* @param pVM Pointer to the VM.
5755	* @param pVCpu Pointer to the VMCPU.
5756	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5757	* out-of-sync. Make sure to update the required fields
5758	* before using them.
5759	*/
5760	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5761	{
5762	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5763
5764	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5765	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5766	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5767	{
5768	/* We need the control registers now, make sure the guest-CPU context is updated. */
5769	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5770	AssertRCReturn(rc3, rc3);
5771
5772	/* Pending HM CR3 sync. */
5773	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5774	{
5775	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5776	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
5777	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
5778	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5779	}
5780
5781	/* Pending HM PAE PDPEs. */
5782	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5783	{
5784	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5785	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5786	}
5787
5788	/* Pending PGM C3 sync. */
5789	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5790	{
5791	int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5792	if (rc2 != VINF_SUCCESS)
5793	{
5794	AssertRC(rc2);
5795	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
5796	return rc2;
5797	}
5798	}
5799
5800	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5801	/* -XXX- what was that about single stepping? */
5802	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5803	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5804	{
5805	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5806	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5807	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
5808	return rc2;
5809	}
5810
5811	/* Pending VM request packets, such as hardware interrupts. */
5812	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5813	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5814	{
5815	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5816	return VINF_EM_PENDING_REQUEST;
5817	}
5818
5819	/* Pending PGM pool flushes. */
5820	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5821	{
5822	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5823	return VINF_PGM_POOL_FLUSH_PENDING;
5824	}
5825
5826	/* Pending DMA requests. */
5827	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5828	{
5829	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5830	return VINF_EM_RAW_TO_R3;
5831	}
5832	}
5833
5834	/* Paranoia. */
5835	return VINF_SUCCESS;
5836	}
5837
5838
5839	/**
5840	* Converts any TRPM trap into a pending HM event. This is typically used when
5841	* entering from ring-3 (not longjmp returns).
5842	*
5843	* @param pVCpu Pointer to the VMCPU.
5844	*/
5845	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
5846	{
5847	Assert(TRPMHasTrap(pVCpu));
5848	Assert(!pVCpu->hm.s.Event.fPending);
5849
5850	uint8_t uVector;
5851	TRPMEVENT enmTrpmEvent;
5852	RTGCUINT uErrCode;
5853	RTGCUINTPTR GCPtrFaultAddress;
5854	uint8_t cbInstr;
5855
5856	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5857	AssertRC(rc);
5858
5859	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5860	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5861	if (enmTrpmEvent == TRPM_TRAP)
5862	{
5863	switch (uVector)
5864	{
5865	case X86_XCPT_BP:
5866	case X86_XCPT_OF:
5867	{
5868	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5869	break;
5870	}
5871
5872	case X86_XCPT_PF:
5873	case X86_XCPT_DF:
5874	case X86_XCPT_TS:
5875	case X86_XCPT_NP:
5876	case X86_XCPT_SS:
5877	case X86_XCPT_GP:
5878	case X86_XCPT_AC:
5879	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5880	/* no break! */
5881	default:
5882	{
5883	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5884	break;
5885	}
5886	}
5887	}
5888	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5889	{
5890	if (uVector == X86_XCPT_NMI)
5891	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5892	else
5893	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5894	}
5895	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
5896	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5897	else
5898	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5899
5900	rc = TRPMResetTrap(pVCpu);
5901	AssertRC(rc);
5902	Log4(("TRPM->HM event: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5903	u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
5904
5905	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
5906	STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
5907	}
5908
5909
5910	/**
5911	* Converts any pending HM event into a TRPM trap. Typically used when leaving
5912	* VT-x to execute any instruction.
5913	*
5914	* @param pvCpu Pointer to the VMCPU.
5915	*/
5916	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
5917	{
5918	Assert(pVCpu->hm.s.Event.fPending);
5919
5920	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5921	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5922	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5923	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5924
5925	/* If a trap was already pending, we did something wrong! */
5926	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5927
5928	TRPMEVENT enmTrapType;
5929	switch (uVectorType)
5930	{
5931	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5932	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5933	enmTrapType = TRPM_HARDWARE_INT;
5934	break;
5935	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5936	enmTrapType = TRPM_SOFTWARE_INT;
5937	break;
5938	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5939	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5940	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5941	enmTrapType = TRPM_TRAP;
5942	break;
5943	default:
5944	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5945	enmTrapType = TRPM_32BIT_HACK;
5946	break;
5947	}
5948
5949	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5950
5951	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5952	AssertRC(rc);
5953
5954	if (fErrorCodeValid)
5955	TRPMSetErrorCode(pVCpu, uErrorCode);
5956
5957	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5958	&& uVector == X86_XCPT_PF)
5959	{
5960	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
5961	}
5962	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5963	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5964	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5965	{
5966	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5967	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
5968	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
5969	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
5970	}
5971	pVCpu->hm.s.Event.fPending = false;
5972	}
5973
5974
5975	/**
5976	* Does the necessary state syncing before doing a longjmp to ring-3.
5977	*
5978	* @param pVM Pointer to the VM.
5979	* @param pVCpu Pointer to the VMCPU.
5980	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5981	* out-of-sync. Make sure to update the required fields
5982	* before using them.
5983	* @param rcExit The reason for exiting to ring-3. Can be
5984	* VINF_VMM_UNKNOWN_RING3_CALL.
5985	*
5986	* @remarks No-long-jmp zone!!!
5987	*/
5988	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5989	{
5990	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5991	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5992
5993	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
5994	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
5995	AssertRC(rc);
5996
5997	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
5998	if (CPUMIsGuestFPUStateActive(pVCpu))
5999	{
6000	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6001	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6002	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
6003	}
6004
6005	/* Restore host debug registers if necessary and resync on next R0 reentry. */
6006	if (CPUMIsGuestDebugStateActive(pVCpu))
6007	{
6008	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
6009	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
6010	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
6011	}
6012	else if (CPUMIsHyperDebugStateActive(pVCpu))
6013	{
6014	CPUMR0LoadHostDebugState(pVM, pVCpu);
6015	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
6016	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6017	}
6018
6019	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6020	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6021	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6022	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6023	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6024	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6025	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6026	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6027
6028	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6029	}
6030
6031
6032	/**
6033	* An action requires us to go back to ring-3. This function does the necessary
6034	* steps before we can safely return to ring-3. This is not the same as longjmps
6035	* to ring-3, this is voluntary.
6036	*
6037	* @param pVM Pointer to the VM.
6038	* @param pVCpu Pointer to the VMCPU.
6039	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6040	* out-of-sync. Make sure to update the required fields
6041	* before using them.
6042	* @param rcExit The reason for exiting to ring-3. Can be
6043	* VINF_VMM_UNKNOWN_RING3_CALL.
6044	*/
6045	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6046	{
6047	Assert(pVM);
6048	Assert(pVCpu);
6049	Assert(pMixedCtx);
6050	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6051
6052	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
6053	{
6054	/* We want to see what the guest-state was before VM-entry, don't resync here, as we won't continue guest execution. */
6055	return;
6056	}
6057	else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6058	{
6059	VMXGetActivateVMCS(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6060	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
6061	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6062	pVCpu->hm.s.vmx.LastError.idCurrentCpu = RTMpCpuId();
6063	return;
6064	}
6065
6066	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6067	VMMRZCallRing3Disable(pVCpu);
6068	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6069
6070	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6071	if (pVCpu->hm.s.Event.fPending)
6072	{
6073	hmR0VmxPendingEventToTrpmTrap(pVCpu);
6074	Assert(!pVCpu->hm.s.Event.fPending);
6075	}
6076
6077	/* Sync. the guest state. */
6078	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
6079	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6080
6081	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6082	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6083	\| CPUM_CHANGED_LDTR
6084	\| CPUM_CHANGED_GDTR
6085	\| CPUM_CHANGED_IDTR
6086	\| CPUM_CHANGED_TR
6087	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
6088
6089	/* On our way back from ring-3 the following needs to be done. */
6090	/** @todo This can change with preemption hooks. */
6091	if (rcExit == VINF_EM_RAW_INTERRUPT)
6092	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
6093	else
6094	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
6095
6096	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6097	VMMRZCallRing3Enable(pVCpu);
6098	}
6099
6100
6101	/**
6102	* VMMRZCallRing3() callback wrapper which saves the guest state before we
6103	* longjump to ring-3 and possibly get preempted.
6104	*
6105	* @param pVCpu Pointer to the VMCPU.
6106	* @param enmOperation The operation causing the ring-3 longjump.
6107	* @param pvUser The user argument (pointer to the possibly
6108	* out-of-date guest-CPU context).
6109	*
6110	* @remarks Must never be called with @a enmOperation ==
6111	* VMMCALLRING3_VM_R0_ASSERTION.
6112	*/
6113	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6114	{
6115	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion. */
6116	Assert(pVCpu);
6117	Assert(pvUser);
6118	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6119	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6120
6121	VMMRZCallRing3Disable(pVCpu);
6122	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6123	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
6124	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
6125	VMMRZCallRing3Enable(pVCpu);
6126	}
6127
6128
6129	/**
6130	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6131	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
6132	*
6133	* @param pVCpu Pointer to the VMCPU.
6134	*/
6135	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6136	{
6137	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6138	{
6139	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6140	{
6141	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6142	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6143	AssertRC(rc);
6144	}
6145	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6146	}
6147
6148
6149	/**
6150	* Injects any pending events into the guest if the guest is in a state to
6151	* receive them.
6152	*
6153	* @returns VBox status code (informational status codes included).
6154	* @param pVCpu Pointer to the VMCPU.
6155	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6156	* out-of-sync. Make sure to update the required fields
6157	* before using them.
6158	*/
6159	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6160	{
6161	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6162	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6163	bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6164	bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6165
6166	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6167	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6168	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6169	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6170	Assert(!TRPMHasTrap(pVCpu));
6171
6172	int rc = VINF_SUCCESS;
6173	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
6174	{
6175	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6176	bool fInject = true;
6177	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6178	{
6179	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6180	AssertRCReturn(rc, rc);
6181	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6182	if ( fBlockInt
6183	\|\| fBlockSti
6184	\|\| fBlockMovSS)
6185	{
6186	fInject = false;
6187	}
6188	}
6189	else if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6190	&& ( fBlockMovSS
6191	\|\| fBlockSti))
6192	{
6193	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6194	fInject = false;
6195	}
6196
6197	if (fInject)
6198	{
6199	Log4(("Injecting pending event vcpu[%RU32]\n", pVCpu->idCpu));
6200	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
6201	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6202	AssertRCReturn(rc, rc);
6203	pVCpu->hm.s.Event.fPending = false;
6204
6205	#ifdef VBOX_WITH_STATISTICS
6206	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6207	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6208	else
6209	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6210	#endif
6211	}
6212	else
6213	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6214	} /** @todo SMI. SMIs take priority over NMIs. */
6215	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6216	{
6217	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6218	if ( !fBlockMovSS
6219	&& !fBlockSti)
6220	{
6221	Log4(("Injecting NMI\n"));
6222	uint32_t u32IntrInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6223	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6224	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
6225	0 /* GCPtrFaultAddress */, &uIntrState);
6226	AssertRCReturn(rc, rc);
6227	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6228
6229	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6230	}
6231	else
6232	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6233	}
6234	else if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
6235	{
6236	/* Check if there are guest external interrupts (PIC/APIC) pending and inject them if the guest can receive them. */
6237	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6238	AssertRCReturn(rc, rc);
6239	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6240	if ( !fBlockInt
6241	&& !fBlockSti
6242	&& !fBlockMovSS)
6243	{
6244	uint8_t u8Interrupt;
6245	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6246	if (RT_SUCCESS(rc))
6247	{
6248	Log4(("Injecting interrupt u8Interrupt=%#x\n", u8Interrupt));
6249	uint32_t u32IntrInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6250	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6251	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
6252	0 /* GCPtrFaultAddress */, &uIntrState);
6253
6254	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6255	}
6256	else
6257	{
6258	/** @todo Does this actually happen? If not turn it into an assertion. */
6259	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
6260	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6261	rc = VINF_SUCCESS;
6262	}
6263	}
6264	else
6265	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6266	}
6267
6268	/*
6269	* Delivery pending debug exception if the guest is single-stepping. The interruptibility-state could have been changed by
6270	* hmR0VmxInjectEventVmcs() (e.g. real-on-v86 injecting software interrupts), re-evaluate it and set the BS bit.
6271	*/
6272	fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6273	fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6274	int rc2 = VINF_SUCCESS;
6275	if ( fBlockSti
6276	\|\| fBlockMovSS)
6277	{
6278	if (!DBGFIsStepping(pVCpu))
6279	{
6280	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6281	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6282	{
6283	/*
6284	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD, VMX_EXIT_MTF
6285	* VMX_EXIT_APIC_WRITE, VMX_EXIT_VIRTUALIZED_EOI. See Intel spec. 27.3.4 "Saving Non-Register State".
6286	*/
6287	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6288	AssertRCReturn(rc, rc);
6289	}
6290	}
6291	else
6292	{
6293	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6294	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6295	uIntrState = 0;
6296	}
6297	}
6298
6299	/*
6300	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6301	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6302	*/
6303	rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6304	AssertRC(rc2);
6305
6306	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6307	return rc;
6308	}
6309
6310
6311	/**
6312	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6313	*
6314	* @param pVCpu Pointer to the VMCPU.
6315	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6316	* out-of-sync. Make sure to update the required fields
6317	* before using them.
6318	*/
6319	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6320	{
6321	uint32_t u32IntrInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6322	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6323	}
6324
6325
6326	/**
6327	* Injects a double-fault (#DF) exception into the VM.
6328	*
6329	* @returns VBox status code (informational status code included).
6330	* @param pVCpu Pointer to the VMCPU.
6331	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6332	* out-of-sync. Make sure to update the required fields
6333	* before using them.
6334	*/
6335	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6336	{
6337	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6338	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6339	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6340	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
6341	puIntrState);
6342	}
6343
6344
6345	/**
6346	* Sets a debug (#DB) exception as pending-for-injection into the VM.
6347	*
6348	* @param pVCpu Pointer to the VMCPU.
6349	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6350	* out-of-sync. Make sure to update the required fields
6351	* before using them.
6352	*/
6353	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6354	{
6355	uint32_t u32IntrInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6356	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6357	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6358	}
6359
6360
6361	/**
6362	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6363	*
6364	* @param pVCpu Pointer to the VMCPU.
6365	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6366	* out-of-sync. Make sure to update the required fields
6367	* before using them.
6368	* @param cbInstr The value of RIP that is to be pushed on the guest
6369	* stack.
6370	*/
6371	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6372	{
6373	uint32_t u32IntrInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6374	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6375	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6376	}
6377
6378
6379	/**
6380	* Injects a general-protection (#GP) fault into the VM.
6381	*
6382	* @returns VBox status code (informational status code included).
6383	* @param pVCpu Pointer to the VMCPU.
6384	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6385	* out-of-sync. Make sure to update the required fields
6386	* before using them.
6387	* @param u32ErrorCode The error code associated with the #GP.
6388	*/
6389	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6390	uint32_t *puIntrState)
6391	{
6392	uint32_t u32IntrInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6393	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6394	if (fErrorCodeValid)
6395	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6396	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
6397	puIntrState);
6398	}
6399
6400
6401	/**
6402	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6403	*
6404	* @param pVCpu Pointer to the VMCPU.
6405	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6406	* out-of-sync. Make sure to update the required fields
6407	* before using them.
6408	* @param uVector The software interrupt vector number.
6409	* @param cbInstr The value of RIP that is to be pushed on the guest
6410	* stack.
6411	*/
6412	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6413	{
6414	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6415	if ( uVector == X86_XCPT_BP
6416	\|\| uVector == X86_XCPT_OF)
6417	{
6418	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6419	}
6420	else
6421	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6422	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6423	}
6424
6425
6426	/**
6427	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6428	* stack.
6429	*
6430	* @returns VBox status code (information status code included).
6431	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6432	* @param pVM Pointer to the VM.
6433	* @param pMixedCtx Pointer to the guest-CPU context.
6434	* @param uValue The value to push to the guest stack.
6435	*/
6436	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6437	{
6438	/*
6439	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6440	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6441	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6442	*/
6443	if (pMixedCtx->sp == 1)
6444	return VINF_EM_RESET;
6445	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6446	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6447	AssertRCReturn(rc, rc);
6448	return rc;
6449	}
6450
6451
6452	/**
6453	* Injects an event into the guest upon VM-entry by updating the relevant fields
6454	* in the VM-entry area in the VMCS.
6455	*
6456	* @returns VBox status code (informational error codes included).
6457	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6458	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6459	*
6460	* @param pVCpu Pointer to the VMCPU.
6461	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6462	* be out-of-sync. Make sure to update the required
6463	* fields before using them.
6464	* @param u64IntrInfo The VM-entry interruption-information field.
6465	* @param cbInstr The VM-entry instruction length in bytes (for
6466	* software interrupts, exceptions and privileged
6467	* software exceptions).
6468	* @param u32ErrCode The VM-entry exception error code.
6469	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6470	* @param puIntrState Pointer to the current guest interruptibility-state.
6471	* This interruptibility-state will be updated if
6472	* necessary. This cannot not be NULL.
6473	*
6474	* @remarks No-long-jump zone!!!
6475	* @remarks Requires CR0!
6476	*/
6477	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6478	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6479	{
6480	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6481	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6482	Assert(puIntrState);
6483	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6484
6485	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6486	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6487
6488	#ifdef VBOX_STRICT
6489	/* Validate the error-code-valid bit for hardware exceptions. */
6490	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
6491	{
6492	switch (uVector)
6493	{
6494	case X86_XCPT_PF:
6495	case X86_XCPT_DF:
6496	case X86_XCPT_TS:
6497	case X86_XCPT_NP:
6498	case X86_XCPT_SS:
6499	case X86_XCPT_GP:
6500	case X86_XCPT_AC:
6501	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo),
6502	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
6503	/* fallthru */
6504	default:
6505	break;
6506	}
6507	}
6508	#endif
6509
6510	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6511	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6512	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6513
6514	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6515
6516	/* We require CR0 to check if the guest is in real-mode. */
6517	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6518	AssertRCReturn(rc, rc);
6519
6520	/*
6521	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6522	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6523	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6524	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6525	*/
6526	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6527	{
6528	PVM pVM = pVCpu->CTX_SUFF(pVM);
6529	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6530	{
6531	Assert(PDMVmmDevHeapIsEnabled(pVM));
6532	Assert(pVM->hm.s.vmx.pRealModeTSS);
6533
6534	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6535	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6536	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6537	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6538	AssertRCReturn(rc, rc);
6539	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6540
6541	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6542	const size_t cbIdtEntry = 4;
6543	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6544	{
6545	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6546	if (uVector == X86_XCPT_DF)
6547	return VINF_EM_RESET;
6548	else if (uVector == X86_XCPT_GP)
6549	{
6550	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6551	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6552	}
6553
6554	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6555	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6556	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6557	}
6558
6559	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6560	uint16_t uGuestIp = pMixedCtx->ip;
6561	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6562	{
6563	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6564	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6565	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6566	}
6567	else if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6568	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6569
6570	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6571	uint16_t offIdtEntry = 0;
6572	RTSEL selIdtEntry = 0;
6573	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6574	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6575	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6576	AssertRCReturn(rc, rc);
6577
6578	/* Construct the stack frame for the interrupt/exception handler. */
6579	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6580	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6581	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6582	AssertRCReturn(rc, rc);
6583
6584	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6585	if (rc == VINF_SUCCESS)
6586	{
6587	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6588	pMixedCtx->rip = offIdtEntry;
6589	pMixedCtx->cs.Sel = selIdtEntry;
6590	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6591	if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6592	&& uVector == X86_XCPT_PF)
6593	{
6594	pMixedCtx->cr2 = GCPtrFaultAddress;
6595	}
6596	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6597	\| HM_CHANGED_GUEST_RIP
6598	\| HM_CHANGED_GUEST_RFLAGS
6599	\| HM_CHANGED_GUEST_RSP;
6600
6601	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6602	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6603	{
6604	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6605	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6606	Log4(("Clearing inhibition due to STI.\n"));
6607	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6608	}
6609	Log4(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6610	}
6611	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6612	return rc;
6613	}
6614	else
6615	{
6616	/*
6617	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6618	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6619	*/
6620	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6621	}
6622	}
6623
6624	/* Validate. */
6625	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6626	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6627	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6628
6629	/* Inject. */
6630	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6631	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6632	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6633	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6634
6635	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6636	&& uVector == X86_XCPT_PF)
6637	{
6638	pMixedCtx->cr2 = GCPtrFaultAddress;
6639	}
6640
6641	Log4(("Injecting vcpu[%RU32] u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
6642	u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6643
6644	AssertRCReturn(rc, rc);
6645	return rc;
6646	}
6647
6648
6649	/**
6650	* Enters the VT-x session.
6651	*
6652	* @returns VBox status code.
6653	* @param pVM Pointer to the VM.
6654	* @param pVCpu Pointer to the VMCPU.
6655	* @param pCpu Pointer to the CPU info struct.
6656	*/
6657	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6658	{
6659	AssertPtr(pVM);
6660	AssertPtr(pVCpu);
6661	Assert(pVM->hm.s.vmx.fSupported);
6662	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6663	NOREF(pCpu);
6664
6665	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6666
6667	#ifdef VBOX_STRICT
6668	/* Make sure we're in VMX root mode. */
6669	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6670	if (!(u32HostCR4 & X86_CR4_VMXE))
6671	{
6672	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6673	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6674	}
6675	#endif
6676
6677	/* Load the active VMCS as the current one. */
6678	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6679	if (RT_FAILURE(rc))
6680	return rc;
6681
6682	/** @todo this will change with preemption hooks where can VMRESUME as long
6683	* as we're no preempted. */
6684	pVCpu->hm.s.fResumeVM = false;
6685	return VINF_SUCCESS;
6686	}
6687
6688
6689	/**
6690	* Leaves the VT-x session.
6691	*
6692	* @returns VBox status code.
6693	* @param pVM Pointer to the VM.
6694	* @param pVCpu Pointer to the VMCPU.
6695	* @param pCtx Pointer to the guest-CPU context.
6696	*/
6697	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6698	{
6699	AssertPtr(pVCpu);
6700	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6701	NOREF(pVM);
6702	NOREF(pCtx);
6703
6704	/** @todo this will change with preemption hooks where we only VMCLEAR when
6705	* we are actually going to be preempted, not all the time like we
6706	* currently do. */
6707
6708	/* Restore host-state bits that VT-x only restores partially. */
6709	if (pVCpu->hm.s.vmx.fRestoreHostFlags)
6710	{
6711	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6712	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6713	}
6714
6715	/*
6716	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6717	* and mark the VMCS launch-state as "clear".
6718	*/
6719	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6720	return rc;
6721	}
6722
6723
6724	/**
6725	* Saves the host state in the VMCS host-state.
6726	* Sets up the VM-exit MSR-load area.
6727	*
6728	* The CPU state will be loaded from these fields on every successful VM-exit.
6729	*
6730	* @returns VBox status code.
6731	* @param pVM Pointer to the VM.
6732	* @param pVCpu Pointer to the VMCPU.
6733	*
6734	* @remarks No-long-jump zone!!!
6735	*/
6736	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6737	{
6738	AssertPtr(pVM);
6739	AssertPtr(pVCpu);
6740	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6741
6742	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6743
6744	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6745	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6746	return VINF_SUCCESS;
6747
6748	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6749	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6750
6751	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6752	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6753
6754	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6755	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6756
6757	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6758	return rc;
6759	}
6760
6761
6762	/**
6763	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6764	* loaded from these fields on every successful VM-entry.
6765	*
6766	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6767	* Sets up the VM-entry controls.
6768	* Sets up the appropriate VMX non-root function to execute guest code based on
6769	* the guest CPU mode.
6770	*
6771	* @returns VBox status code.
6772	* @param pVM Pointer to the VM.
6773	* @param pVCpu Pointer to the VMCPU.
6774	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6775	* out-of-sync. Make sure to update the required fields
6776	* before using them.
6777	*
6778	* @remarks No-long-jump zone!!!
6779	*/
6780	static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6781	{
6782	AssertPtr(pVM);
6783	AssertPtr(pVCpu);
6784	AssertPtr(pMixedCtx);
6785	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6786
6787	#ifdef LOG_ENABLED
6788	/** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
6789	* probably not initialized yet? Anyway this will do for now. */
6790	bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
6791	VMMR0LogFlushDisable(pVCpu);
6792	#endif
6793
6794	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6795
6796	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6797
6798	/* Determine real-on-v86 mode. */
6799	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6800	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6801	&& CPUMIsGuestInRealModeEx(pMixedCtx))
6802	{
6803	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6804	}
6805
6806	/*
6807	* Load the guest-state into the VMCS.
6808	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
6809	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
6810	*/
6811	int rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
6812	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6813
6814	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
6815	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6816
6817	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
6818	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6819
6820	rc = hmR0VmxLoadGuestControlRegs(pVCpu, pMixedCtx);
6821	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6822
6823	/* Must be done after CR0 is loaded (strict builds require CR0 for segment register validation checks). */
6824	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
6825	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6826
6827	rc = hmR0VmxLoadGuestDebugRegs(pVCpu, pMixedCtx);
6828	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6829
6830	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
6831	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6832
6833	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
6834	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6835
6836	/* Must be done after hmR0VmxLoadGuestDebugRegs() as it may update eflags.TF for debugging purposes. */
6837	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
6838	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6839
6840	rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
6841	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6842
6843	/* Clear any unused and reserved bits. */
6844	pVCpu->hm.s.fContextUseFlags &= ~( HM_CHANGED_GUEST_CR2
6845	\| HM_CHANGED_GUEST_MSR /* legacy */);
6846
6847	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6848	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p idCpu=%RU32 fContextUseFlags=%#RX32\n",
6849	pVM, pVCpu, pVCpu->idCpu, pVCpu->hm.s.fContextUseFlags));
6850
6851	#ifdef LOG_ENABLED
6852	/* Only reenable log-flushing if the caller has it enabled. */
6853	if (!fCallerDisabledLogFlush)
6854	VMMR0LogFlushEnable(pVCpu);
6855	#endif
6856
6857	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6858	return rc;
6859	}
6860
6861
6862	/**
6863	* Loads the guest state into the VMCS guest-state area.
6864	*
6865	* @returns VBox status code.
6866	* @param pVM Pointer to the VM.
6867	* @param pVCpu Pointer to the VMCPU.
6868	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6869	* out-of-sync. Make sure to update the required fields
6870	* before using them.
6871	*
6872	* @remarks No-long-jump zone!!!
6873	*/
6874	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6875	{
6876	/*
6877	* Avoid reloading the guest state on longjmp reentrants and do it lazily just before executing the guest.
6878	* This only helps when we get rescheduled more than once to a different host CPU on a longjmp trip before
6879	* finally executing guest code.
6880	*/
6881	return VINF_SUCCESS;
6882	}
6883
6884
6885	/**
6886	* Does the preparations before executing guest code in VT-x.
6887	*
6888	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6889	* recompiler. We must be cautious what we do here regarding committing
6890	* guest-state information into the VMCS assuming we assuredly execute the
6891	* guest in VT-x. If we fall back to the recompiler after updating the VMCS and
6892	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6893	* that the recompiler can (and should) use them when it resumes guest
6894	* execution. Otherwise such operations must be done when we can no longer
6895	* exit to ring-3.
6896	*
6897	* @returns VBox status code (informational status codes included).
6898	* @retval VINF_SUCCESS if we can proceed with running the guest.
6899	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6900	* into the guest.
6901	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6902	*
6903	* @param pVM Pointer to the VM.
6904	* @param pVCpu Pointer to the VMCPU.
6905	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6906	* out-of-sync. Make sure to update the required fields
6907	* before using them.
6908	* @param pVmxTransient Pointer to the VMX transient structure.
6909	*
6910	* @remarks Called with preemption disabled.
6911	*/
6912	static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6913	{
6914	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6915
6916	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6917	PGMRZDynMapFlushAutoSet(pVCpu);
6918	#endif
6919
6920	/* Check force flag actions that might require us to go back to ring-3. */
6921	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6922	if (rc != VINF_SUCCESS)
6923	return rc;
6924
6925	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6926	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6927	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6928	{
6929	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6930	RTGCPHYS GCPhysApicBase;
6931	GCPhysApicBase = pMixedCtx->msrApicBase;
6932	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6933
6934	/* Unalias any existing mapping. */
6935	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6936	AssertRCReturn(rc, rc);
6937
6938	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6939	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6940	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6941	AssertRCReturn(rc, rc);
6942
6943	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6944	}
6945
6946	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6947	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6948	pVmxTransient->uEFlags = ASMIntDisableFlags();
6949	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6950	{
6951	ASMSetFlags(pVmxTransient->uEFlags);
6952	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6953	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6954	return VINF_EM_RAW_INTERRUPT;
6955	}
6956	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6957	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6958	#endif
6959
6960	/*
6961	* Evaluates and injects any pending events, toggling force-flags and updating the guest-interruptibility
6962	* state (interrupt shadow) in the VMCS. This -can- potentially be reworked to be done before disabling
6963	* interrupts and handle returning to ring-3 afterwards, but requires very careful state restoration.
6964	*/
6965	/** @todo Rework event evaluation and injection to be completely separate. */
6966	if (TRPMHasTrap(pVCpu))
6967	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
6968
6969	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
6970	AssertRCReturn(rc, rc);
6971	return rc;
6972	}
6973
6974
6975	/**
6976	* Prepares to run guest code in VT-x and we've committed to doing so. This
6977	* means there is no backing out to ring-3 or anywhere else at this
6978	* point.
6979	*
6980	* @param pVM Pointer to the VM.
6981	* @param pVCpu Pointer to the VMCPU.
6982	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6983	* out-of-sync. Make sure to update the required fields
6984	* before using them.
6985	* @param pVmxTransient Pointer to the VMX transient structure.
6986	*
6987	* @remarks Called with preemption disabled.
6988	* @remarks No-long-jump zone!!!
6989	*/
6990	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6991	{
6992	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6993	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6994
6995	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6996	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6997	pVmxTransient->uEFlags = ASMIntDisableFlags();
6998	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6999	#endif
7000
7001	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
7002	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
7003	Log5(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
7004	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7005	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
7006	#endif
7007	int rc = VINF_SUCCESS;
7008	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
7009	{
7010	rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7011	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7012	}
7013	else if (pVCpu->hm.s.fContextUseFlags)
7014	{
7015	rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7016	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7017	}
7018	AssertRC(rc);
7019	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
7020
7021	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
7022	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7023	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
7024
7025	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
7026	\|\| HMR0GetCurrentCpu()->idCpu != pVCpu->hm.s.idLastCpu)
7027	{
7028	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
7029	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
7030	}
7031
7032	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
7033	hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
7034	Assert(HMR0GetCurrentCpu()->idCpu == pVCpu->hm.s.idLastCpu);
7035
7036	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
7037
7038	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
7039	to start executing. */
7040
7041	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7042	/*
7043	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
7044	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
7045	*/
7046	if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7047	&& !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7048	{
7049	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
7050	uint64_t u64HostTscAux = 0;
7051	int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
7052	AssertRC(rc2);
7053	ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
7054	}
7055	#endif
7056	}
7057
7058
7059	/**
7060	* Performs some essential restoration of state after running guest code in
7061	* VT-x.
7062	*
7063	* @param pVM Pointer to the VM.
7064	* @param pVCpu Pointer to the VMCPU.
7065	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7066	* out-of-sync. Make sure to update the required fields
7067	* before using them.
7068	* @param pVmxTransient Pointer to the VMX transient structure.
7069	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
7070	*
7071	* @remarks Called with interrupts disabled.
7072	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
7073	* unconditionally when it is safe to do so.
7074	*/
7075	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
7076	{
7077	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7078
7079	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
7080	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
7081	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
7082	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
7083	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
7084
7085	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7086	{
7087	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7088	/* Restore host's TSC_AUX. */
7089	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7090	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
7091	#endif
7092	/** @todo Find a way to fix hardcoding a guestimate. */
7093	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
7094	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
7095	}
7096
7097	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
7098	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
7099	Assert(!(ASMGetFlags() & X86_EFL_IF));
7100	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7101
7102	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
7103	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
7104
7105	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
7106	uint32_t uExitReason;
7107	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
7108	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7109	AssertRC(rc);
7110	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
7111	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
7112
7113	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
7114	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
7115
7116	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
7117	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
7118	{
7119	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
7120	pVmxTransient->fVMEntryFailed));
7121	return;
7122	}
7123
7124	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
7125	{
7126	/* Update the guest interruptibility-state from the VMCS. */
7127	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
7128	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
7129	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7130	AssertRC(rc);
7131	#endif
7132	/*
7133	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
7134	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
7135	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
7136	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
7137	*/
7138	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7139	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
7140	{
7141	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
7142	AssertRC(rc);
7143	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7144	}
7145	}
7146	}
7147
7148
7149	/**
7150	* Runs the guest code using VT-x.
7151	*
7152	* @returns VBox status code.
7153	* @param pVM Pointer to the VM.
7154	* @param pVCpu Pointer to the VMCPU.
7155	* @param pCtx Pointer to the guest-CPU context.
7156	*
7157	* @remarks Called with preemption disabled.
7158	*/
7159	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7160	{
7161	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7162	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7163
7164	VMXTRANSIENT VmxTransient;
7165	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7166	int rc = VERR_INTERNAL_ERROR_5;
7167	uint32_t cLoops = 0;
7168
7169	for (;; cLoops++)
7170	{
7171	Assert(!HMR0SuspendPending());
7172	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
7173	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
7174	(unsigned)RTMpCpuId(), cLoops));
7175
7176	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
7177	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7178	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7179	if (rc != VINF_SUCCESS)
7180	break;
7181
7182	/*
7183	* No longjmps to ring-3 from this point on!!!
7184	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7185	* This also disables flushing of the R0-logger instance (if any).
7186	*/
7187	VMMRZCallRing3Disable(pVCpu);
7188	VMMRZCallRing3RemoveNotification(pVCpu);
7189	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7190
7191	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7192	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7193
7194	/*
7195	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
7196	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
7197	*/
7198	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7199	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7200	{
7201	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7202	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7203	return rc;
7204	}
7205
7206	/* Handle the VM-exit. */
7207	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7208	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7209	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7210	HMVMX_START_EXIT_DISPATCH_PROF();
7211	#ifdef HMVMX_USE_FUNCTION_TABLE
7212	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7213	#else
7214	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7215	#endif
7216	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7217	if (rc != VINF_SUCCESS)
7218	break;
7219	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7220	{
7221	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7222	rc = VINF_EM_RAW_INTERRUPT;
7223	break;
7224	}
7225	}
7226
7227	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7228	if (rc == VERR_EM_INTERPRETER)
7229	rc = VINF_EM_RAW_EMULATE_INSTR;
7230	else if (rc == VINF_EM_RESET)
7231	rc = VINF_EM_TRIPLE_FAULT;
7232	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
7233	return rc;
7234	}
7235
7236
7237	#ifndef HMVMX_USE_FUNCTION_TABLE
7238	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
7239	{
7240	int rc;
7241	switch (rcReason)
7242	{
7243	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
7244	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
7245	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
7246	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
7247	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
7248	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
7249	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7250	case VMX_EXIT_XCPT_OR_NMI: rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); break;
7251	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
7252	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
7253	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7254	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
7255	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
7256	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
7257	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
7258	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7259	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7260	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
7261	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
7262	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
7263	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
7264	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
7265	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
7266	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
7267	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
7268	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7269	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7270	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
7271	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
7272	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
7273	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
7274	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
7275	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
7276
7277	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
7278	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7279	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
7280	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
7281	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7282	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7283	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
7284	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
7285	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
7286
7287	case VMX_EXIT_VMCALL:
7288	case VMX_EXIT_VMCLEAR:
7289	case VMX_EXIT_VMLAUNCH:
7290	case VMX_EXIT_VMPTRLD:
7291	case VMX_EXIT_VMPTRST:
7292	case VMX_EXIT_VMREAD:
7293	case VMX_EXIT_VMRESUME:
7294	case VMX_EXIT_VMWRITE:
7295	case VMX_EXIT_VMXOFF:
7296	case VMX_EXIT_VMXON:
7297	case VMX_EXIT_INVEPT:
7298	case VMX_EXIT_INVVPID:
7299	case VMX_EXIT_VMFUNC:
7300	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
7301	break;
7302	default:
7303	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
7304	break;
7305	}
7306	return rc;
7307	}
7308	#endif
7309
7310	#ifdef DEBUG
7311	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
7312	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
7313	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
7314
7315	# define HMVMX_ASSERT_PREEMPT_CPUID() \
7316	do \
7317	{ \
7318	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
7319	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
7320	} while (0)
7321
7322	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
7323	do { \
7324	AssertPtr(pVCpu); \
7325	AssertPtr(pMixedCtx); \
7326	AssertPtr(pVmxTransient); \
7327	Assert(pVmxTransient->fVMEntryFailed == false); \
7328	Assert(ASMIntAreEnabled()); \
7329	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7330	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
7331	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
7332	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7333	if (VMMR0IsLogFlushDisabled(pVCpu)) \
7334	HMVMX_ASSERT_PREEMPT_CPUID(); \
7335	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
7336	} while (0)
7337
7338	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
7339	do { \
7340	Log4Func(("\n")); \
7341	} while(0)
7342	#else /* Release builds */
7343	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
7344	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
7345	#endif
7346
7347
7348	/**
7349	* Advances the guest RIP after reading it from the VMCS.
7350	*
7351	* @returns VBox status code.
7352	* @param pVCpu Pointer to the VMCPU.
7353	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7354	* out-of-sync. Make sure to update the required fields
7355	* before using them.
7356	* @param pVmxTransient Pointer to the VMX transient structure.
7357	*
7358	* @remarks No-long-jump zone!!!
7359	*/
7360	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7361	{
7362	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7363	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7364	AssertRCReturn(rc, rc);
7365
7366	pMixedCtx->rip += pVmxTransient->cbInstr;
7367	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7368	return rc;
7369	}
7370
7371
7372	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
7373	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
7374	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
7375
7376	/** @name VM-exit handlers.
7377	* @{
7378	*/
7379
7380	/**
7381	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
7382	*/
7383	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7384	{
7385	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7386	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
7387	/* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
7388	#if HC_ARCH_BITS == 64 && defined(VBOX_WITH_VMMR0_DISABLE_PREEMPTION)
7389	Assert(ASMIntAreEnabled());
7390	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUsePreemptTimer)
7391	return VINF_SUCCESS;
7392	#endif
7393	return VINF_EM_RAW_INTERRUPT;
7394	}
7395
7396
7397	/**
7398	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
7399	*/
7400	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7401	{
7402	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7403	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
7404
7405	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
7406	AssertRCReturn(rc, rc);
7407
7408	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
7409	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
7410	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7411	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo));
7412
7413	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7414	{
7415	/*
7416	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
7417	* anything we inject is not going to cause a VM-exit directly for the event being injected.
7418	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
7419	*
7420	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
7421	*/
7422	VMXDispatchHostNmi();
7423	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmi);
7424	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7425	return VINF_SUCCESS;
7426	}
7427
7428	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
7429	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
7430	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
7431	{
7432	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7433	return VINF_SUCCESS;
7434	}
7435	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
7436	{
7437	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7438	return rc;
7439	}
7440
7441	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
7442	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
7443	switch (uIntrType)
7444	{
7445	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
7446	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7447	/* no break */
7448	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
7449	{
7450	switch (uVector)
7451	{
7452	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
7453	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
7454	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
7455	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
7456	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
7457	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
7458	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
7459	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
7460	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7461	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
7462	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7463	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7464	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7465	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
7466	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7467	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
7468	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7469	#endif
7470	default:
7471	{
7472	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7473	AssertRCReturn(rc, rc);
7474
7475	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
7476	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7477	{
7478	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
7479	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
7480	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7481	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
7482	AssertRCReturn(rc, rc);
7483	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
7484	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
7485	0 /* GCPtrFaultAddress */);
7486	AssertRCReturn(rc, rc);
7487	}
7488	else
7489	{
7490	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
7491	pVCpu->hm.s.u32HMError = uVector;
7492	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7493	}
7494	break;
7495	}
7496	}
7497	break;
7498	}
7499
7500	default:
7501	{
7502	pVCpu->hm.s.u32HMError = uExitIntrInfo;
7503	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
7504	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
7505	break;
7506	}
7507	}
7508	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7509	return rc;
7510	}
7511
7512
7513	/**
7514	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
7515	*/
7516	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7517	{
7518	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7519
7520	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
7521	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7522	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7523	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7524	AssertRCReturn(rc, rc);
7525
7526	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
7527	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
7528	return VINF_SUCCESS;
7529	}
7530
7531
7532	/**
7533	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
7534	*/
7535	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7536	{
7537	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7538	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
7539	pVCpu->hm.s.u32HMError = VMX_EXIT_NMI_WINDOW;
7540	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7541	}
7542
7543
7544	/**
7545	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
7546	*/
7547	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7548	{
7549	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7550	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
7551	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7552	}
7553
7554
7555	/**
7556	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
7557	*/
7558	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7559	{
7560	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7561	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
7562	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7563	}
7564
7565
7566	/**
7567	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
7568	*/
7569	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7570	{
7571	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7572	PVM pVM = pVCpu->CTX_SUFF(pVM);
7573	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7574	if (RT_LIKELY(rc == VINF_SUCCESS))
7575	{
7576	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7577	Assert(pVmxTransient->cbInstr == 2);
7578	}
7579	else
7580	{
7581	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
7582	rc = VERR_EM_INTERPRETER;
7583	}
7584	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
7585	return rc;
7586	}
7587
7588
7589	/**
7590	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
7591	*/
7592	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7593	{
7594	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7595	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
7596	AssertRCReturn(rc, rc);
7597
7598	if (pMixedCtx->cr4 & X86_CR4_SMXE)
7599	return VINF_EM_RAW_EMULATE_INSTR;
7600
7601	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
7602	pVCpu->hm.s.u32HMError = VMX_EXIT_GETSEC;
7603	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7604	}
7605
7606
7607	/**
7608	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
7609	*/
7610	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7611	{
7612	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7613	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7614	AssertRCReturn(rc, rc);
7615
7616	PVM pVM = pVCpu->CTX_SUFF(pVM);
7617	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7618	if (RT_LIKELY(rc == VINF_SUCCESS))
7619	{
7620	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7621	Assert(pVmxTransient->cbInstr == 2);
7622	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7623	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
7624	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7625	}
7626	else
7627	{
7628	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
7629	rc = VERR_EM_INTERPRETER;
7630	}
7631	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7632	return rc;
7633	}
7634
7635
7636	/**
7637	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7638	*/
7639	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7640	{
7641	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7642	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7643	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
7644	AssertRCReturn(rc, rc);
7645
7646	PVM pVM = pVCpu->CTX_SUFF(pVM);
7647	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
7648	if (RT_LIKELY(rc == VINF_SUCCESS))
7649	{
7650	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7651	Assert(pVmxTransient->cbInstr == 3);
7652	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7653	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
7654	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7655	}
7656	else
7657	{
7658	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
7659	rc = VERR_EM_INTERPRETER;
7660	}
7661	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7662	return rc;
7663	}
7664
7665
7666	/**
7667	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7668	*/
7669	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7670	{
7671	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7672	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7673	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
7674	AssertRCReturn(rc, rc);
7675
7676	PVM pVM = pVCpu->CTX_SUFF(pVM);
7677	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7678	if (RT_LIKELY(rc == VINF_SUCCESS))
7679	{
7680	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7681	Assert(pVmxTransient->cbInstr == 2);
7682	}
7683	else
7684	{
7685	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7686	rc = VERR_EM_INTERPRETER;
7687	}
7688	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
7689	return rc;
7690	}
7691
7692
7693	/**
7694	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7695	*/
7696	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7697	{
7698	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7699	PVM pVM = pVCpu->CTX_SUFF(pVM);
7700	Assert(!pVM->hm.s.fNestedPaging);
7701
7702	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7703	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7704	AssertRCReturn(rc, rc);
7705
7706	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
7707	rc = VBOXSTRICTRC_VAL(rc2);
7708	if (RT_LIKELY(rc == VINF_SUCCESS))
7709	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7710	else
7711	{
7712	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
7713	pVmxTransient->uExitQualification, rc));
7714	}
7715	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
7716	return rc;
7717	}
7718
7719
7720	/**
7721	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7722	*/
7723	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7724	{
7725	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7726	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7727	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7728	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7729	AssertRCReturn(rc, rc);
7730
7731	PVM pVM = pVCpu->CTX_SUFF(pVM);
7732	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7733	if (RT_LIKELY(rc == VINF_SUCCESS))
7734	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7735	else
7736	{
7737	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7738	rc = VERR_EM_INTERPRETER;
7739	}
7740	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7741	return rc;
7742	}
7743
7744
7745	/**
7746	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7747	*/
7748	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7749	{
7750	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7751	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7752	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7753	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7754	AssertRCReturn(rc, rc);
7755
7756	PVM pVM = pVCpu->CTX_SUFF(pVM);
7757	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7758	rc = VBOXSTRICTRC_VAL(rc2);
7759	if (RT_LIKELY( rc == VINF_SUCCESS
7760	\|\| rc == VINF_EM_HALT))
7761	{
7762	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7763	AssertRCReturn(rc3, rc3);
7764
7765	if ( rc == VINF_EM_HALT
7766	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7767	{
7768	rc = VINF_SUCCESS;
7769	}
7770	}
7771	else
7772	{
7773	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7774	rc = VERR_EM_INTERPRETER;
7775	}
7776	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7777	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7778	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7779	return rc;
7780	}
7781
7782
7783	/**
7784	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7785	*/
7786	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7787	{
7788	/*
7789	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7790	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7791	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
7792	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7793	*/
7794	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7795	pVCpu->hm.s.u32HMError = VMX_EXIT_RSM;
7796	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7797	}
7798
7799
7800	/**
7801	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7802	*/
7803	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7804	{
7805	/*
7806	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7807	* root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
7808	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
7809	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7810	*/
7811	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7812	pVCpu->hm.s.u32HMError = VMX_EXIT_SMI;
7813	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7814	}
7815
7816
7817	/**
7818	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7819	*/
7820	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7821	{
7822	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7823	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7824	pVCpu->hm.s.u32HMError = VMX_EXIT_IO_SMI;
7825	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7826	}
7827
7828
7829	/**
7830	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7831	*/
7832	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7833	{
7834	/*
7835	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7836	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7837	* See Intel spec. 25.3 "Other Causes of VM-exits".
7838	*/
7839	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7840	pVCpu->hm.s.u32HMError = VMX_EXIT_SIPI;
7841	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7842	}
7843
7844
7845	/**
7846	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7847	* VM-exit.
7848	*/
7849	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7850	{
7851	/*
7852	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
7853	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
7854	*
7855	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
7856	* See Intel spec. "23.8 Restrictions on VMX operation".
7857	*/
7858	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7859	return VINF_SUCCESS;
7860	}
7861
7862
7863	/**
7864	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7865	* VM-exit.
7866	*/
7867	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7868	{
7869	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7870	return VINF_EM_RESET;
7871	}
7872
7873
7874	/**
7875	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7876	*/
7877	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7878	{
7879	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7880	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
7881	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7882	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7883	AssertRCReturn(rc, rc);
7884
7885	pMixedCtx->rip++;
7886	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7887	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7888	rc = VINF_SUCCESS;
7889	else
7890	rc = VINF_EM_HALT;
7891
7892	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7893	return rc;
7894	}
7895
7896
7897	/**
7898	* VM-exit handler for instructions that result in a #UD exception delivered to
7899	* the guest.
7900	*/
7901	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7902	{
7903	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7904	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
7905	return VINF_SUCCESS;
7906	}
7907
7908
7909	/**
7910	* VM-exit handler for expiry of the VMX preemption timer.
7911	*/
7912	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7913	{
7914	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7915
7916	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
7917	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7918
7919	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7920	PVM pVM = pVCpu->CTX_SUFF(pVM);
7921	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7922	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7923	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7924	}
7925
7926
7927	/**
7928	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7929	*/
7930	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7931	{
7932	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7933
7934	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7935	/** @todo check if XSETBV is supported by the recompiler. */
7936	return VERR_EM_INTERPRETER;
7937	}
7938
7939
7940	/**
7941	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7942	*/
7943	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7944	{
7945	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7946
7947	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7948	/** @todo implement EMInterpretInvpcid() */
7949	return VERR_EM_INTERPRETER;
7950	}
7951
7952
7953	/**
7954	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7955	* Error VM-exit.
7956	*/
7957	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7958	{
7959	uint32_t uIntrState;
7960	HMVMXHCUINTREG uHCReg;
7961	uint64_t u64Val;
7962	uint32_t u32Val;
7963
7964	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7965	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7966	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
7967	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7968	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7969	AssertRCReturn(rc, rc);
7970
7971	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7972	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7973	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7974	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7975
7976	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
7977	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
7978	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7979	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7980	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7981	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7982	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7983	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7984	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7985	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7986	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7987	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7988
7989	PVM pVM = pVCpu->CTX_SUFF(pVM);
7990	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7991
7992	return VERR_VMX_INVALID_GUEST_STATE;
7993	}
7994
7995
7996	/**
7997	* VM-exit handler for VM-entry failure due to an MSR-load
7998	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7999	*/
8000	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8001	{
8002	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8003	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8004	}
8005
8006
8007	/**
8008	* VM-exit handler for VM-entry failure due to a machine-check event
8009	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
8010	*/
8011	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8012	{
8013	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8014	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8015	}
8016
8017
8018	/**
8019	* VM-exit handler for all undefined reasons. Should never ever happen.. in
8020	* theory.
8021	*/
8022	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8023	{
8024	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
8025	return VERR_VMX_UNDEFINED_EXIT_CODE;
8026	}
8027
8028
8029	/**
8030	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
8031	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
8032	* Conditional VM-exit.
8033	*/
8034	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8035	{
8036	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8037
8038	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
8039	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
8040	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
8041	return VERR_EM_INTERPRETER;
8042	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8043	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8044	}
8045
8046
8047	/**
8048	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
8049	*/
8050	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8051	{
8052	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8053
8054	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
8055	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
8056	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
8057	return VERR_EM_INTERPRETER;
8058	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8059	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8060	}
8061
8062
8063	/**
8064	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
8065	*/
8066	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8067	{
8068	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8069
8070	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
8071	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8072	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8073	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8074	AssertRCReturn(rc, rc);
8075
8076	PVM pVM = pVCpu->CTX_SUFF(pVM);
8077	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8078	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
8079	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
8080	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
8081
8082	if (RT_LIKELY(rc == VINF_SUCCESS))
8083	{
8084	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8085	Assert(pVmxTransient->cbInstr == 2);
8086	}
8087	return rc;
8088	}
8089
8090
8091	/**
8092	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
8093	*/
8094	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8095	{
8096	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8097	PVM pVM = pVCpu->CTX_SUFF(pVM);
8098	int rc = VINF_SUCCESS;
8099
8100	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
8101	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8102	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8103	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8104	AssertRCReturn(rc, rc);
8105	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
8106
8107	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8108	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
8109	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
8110
8111	if (RT_LIKELY(rc == VINF_SUCCESS))
8112	{
8113	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8114
8115	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
8116	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
8117	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
8118	{
8119	/* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
8120	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
8121	EMInterpretWrmsr() changes it. */
8122	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
8123	}
8124	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
8125	{
8126	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8127	AssertRCReturn(rc, rc);
8128	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
8129	}
8130	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
8131	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8132
8133	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
8134	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
8135	{
8136	switch (pMixedCtx->ecx)
8137	{
8138	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
8139	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
8140	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
8141	case MSR_K8_FS_BASE: /* no break */
8142	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS; break;
8143	case MSR_K8_KERNEL_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS; break;
8144	}
8145	}
8146	#ifdef VBOX_STRICT
8147	else
8148	{
8149	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
8150	switch (pMixedCtx->ecx)
8151	{
8152	case MSR_IA32_SYSENTER_CS:
8153	case MSR_IA32_SYSENTER_EIP:
8154	case MSR_IA32_SYSENTER_ESP:
8155	case MSR_K8_FS_BASE:
8156	case MSR_K8_GS_BASE:
8157	{
8158	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
8159	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8160	}
8161
8162	case MSR_K8_LSTAR:
8163	case MSR_K6_STAR:
8164	case MSR_K8_SF_MASK:
8165	case MSR_K8_TSC_AUX:
8166	case MSR_K8_KERNEL_GS_BASE:
8167	{
8168	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
8169	pMixedCtx->ecx));
8170	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8171	}
8172	}
8173	}
8174	#endif /* VBOX_STRICT */
8175	}
8176	return rc;
8177	}
8178
8179
8180	/**
8181	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
8182	*/
8183	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8184	{
8185	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8186
8187	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
8188	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
8189	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
8190	return VERR_EM_INTERPRETER;
8191	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8192	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8193	}
8194
8195
8196	/**
8197	* VM-exit handler for when the TPR value is lowered below the specified
8198	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
8199	*/
8200	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8201	{
8202	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8203	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
8204
8205	/*
8206	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
8207	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
8208	* resume guest execution.
8209	*/
8210	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
8211	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
8212	return VINF_SUCCESS;
8213	}
8214
8215
8216	/**
8217	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
8218	* VM-exit.
8219	*
8220	* @retval VINF_SUCCESS when guest execution can continue.
8221	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
8222	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
8223	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
8224	* recompiler.
8225	*/
8226	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8227	{
8228	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8229	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
8230	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8231	AssertRCReturn(rc, rc);
8232
8233	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
8234	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
8235	PVM pVM = pVCpu->CTX_SUFF(pVM);
8236	switch (uAccessType)
8237	{
8238	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
8239	{
8240	#if 0
8241	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
8242	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8243	#else
8244	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8245	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8246	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8247	#endif
8248	AssertRCReturn(rc, rc);
8249
8250	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8251	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
8252	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
8253	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
8254
8255	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
8256	{
8257	case 0: /* CR0 */
8258	Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
8259	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8260	break;
8261	case 2: /* C2 **/
8262	/* Nothing to do here, CR2 it's not part of the VMCS. */
8263	break;
8264	case 3: /* CR3 */
8265	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
8266	Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
8267	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
8268	break;
8269	case 4: /* CR4 */
8270	Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
8271	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
8272	break;
8273	case 8: /* CR8 */
8274	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
8275	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
8276	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
8277	break;
8278	default:
8279	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
8280	break;
8281	}
8282
8283	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
8284	break;
8285	}
8286
8287	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
8288	{
8289	/* EMInterpretCRxRead() requires EFER MSR, CS. */
8290	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8291	AssertRCReturn(rc, rc);
8292	Assert( !pVM->hm.s.fNestedPaging
8293	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
8294	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
8295
8296	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
8297	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
8298	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
8299
8300	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8301	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
8302	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
8303	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8304	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
8305	Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
8306	break;
8307	}
8308
8309	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
8310	{
8311	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8312	AssertRCReturn(rc, rc);
8313	rc = EMInterpretCLTS(pVM, pVCpu);
8314	AssertRCReturn(rc, rc);
8315	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8316	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
8317	Log4(("CRX CLTS write rc=%d\n", rc));
8318	break;
8319	}
8320
8321	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
8322	{
8323	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8324	AssertRCReturn(rc, rc);
8325	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
8326	if (RT_LIKELY(rc == VINF_SUCCESS))
8327	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8328	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
8329	Log4(("CRX LMSW write rc=%d\n", rc));
8330	break;
8331	}
8332
8333	default:
8334	{
8335	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
8336	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
8337	}
8338	}
8339
8340	/* Validate possible error codes. */
8341	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
8342	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
8343	if (RT_SUCCESS(rc))
8344	{
8345	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8346	AssertRCReturn(rc2, rc2);
8347	}
8348
8349	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
8350	return rc;
8351	}
8352
8353
8354	/**
8355	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
8356	* VM-exit.
8357	*/
8358	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8359	{
8360	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8361	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
8362
8363	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8364	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8365	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8366	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
8367	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
8368	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
8369	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
8370	AssertRCReturn(rc, rc);
8371
8372	Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8373
8374	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
8375	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
8376	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
8377	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
8378	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
8379	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
8380	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
8381
8382	/* I/O operation lookup arrays. */
8383	static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
8384	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
8385
8386	const uint32_t cbValue = s_aIOSizes[uIOWidth];
8387	const uint32_t cbInstr = pVmxTransient->cbInstr;
8388	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
8389	PVM pVM = pVCpu->CTX_SUFF(pVM);
8390	if (fIOString)
8391	{
8392	/*
8393	* INS/OUTS - I/O String instruction.
8394	*
8395	* Use instruction-information if available, otherwise fall back on
8396	* interpreting the instruction.
8397	*/
8398	#if 0 /* Not quite ready, seem iSegReg assertion trigger once... Do we perhaps need to always read that in longjmp / preempt scenario? */
8399	AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
8400	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.msr.vmx_basic_info))
8401	{
8402	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8403	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
8404	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8405	AssertRCReturn(rc, rc);
8406	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
8407	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
8408	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
8409	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
8410	if (fIOWrite)
8411	{
8412	rc = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
8413	pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
8414	//if (rc == VINF_IOM_R3_IOPORT_WRITE)
8415	// hmR0SavePendingIOPortWriteStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr,
8416	// pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
8417	}
8418	else
8419	{
8420	AssertMsgReturn(pVmxTransient->ExitInstrInfo.StrIo.iSegReg == X86_SREG_ES,
8421	("%#x (%#llx)\n", pVmxTransient->ExitInstrInfo.StrIo.iSegReg, pVmxTransient->ExitInstrInfo.u),
8422	VERR_HMVMX_IPE_4);
8423	rc = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
8424	//if (rc == VINF_IOM_R3_IOPORT_READ)
8425	// hmR0SavePendingIOPortReadStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr);
8426	}
8427	}
8428	else
8429	{
8430	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
8431	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8432	AssertRCReturn(rc, rc);
8433	rc = IEMExecOne(pVCpu);
8434	}
8435	/** @todo IEM needs to be setting these flags somehow. */
8436	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8437	fUpdateRipAlready = true;
8438
8439	#else
8440	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8441	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
8442	if (RT_SUCCESS(rc))
8443	{
8444	if (fIOWrite)
8445	{
8446	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
8447	(DISCPUMODE)pDis->uAddrMode, cbValue);
8448	rc = VBOXSTRICTRC_VAL(rc2);
8449	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
8450	}
8451	else
8452	{
8453	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
8454	(DISCPUMODE)pDis->uAddrMode, cbValue);
8455	rc = VBOXSTRICTRC_VAL(rc2);
8456	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
8457	}
8458	}
8459	else
8460	{
8461	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
8462	rc = VINF_EM_RAW_EMULATE_INSTR;
8463	}
8464	#endif
8465	}
8466	else
8467	{
8468	/*
8469	* IN/OUT - I/O instruction.
8470	*/
8471	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
8472	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
8473	if (fIOWrite)
8474	{
8475	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
8476	rc = VBOXSTRICTRC_VAL(rc2);
8477	if (rc == VINF_IOM_R3_IOPORT_WRITE)
8478	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
8479	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
8480	}
8481	else
8482	{
8483	uint32_t u32Result = 0;
8484	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
8485	rc = VBOXSTRICTRC_VAL(rc2);
8486	if (IOM_SUCCESS(rc))
8487	{
8488	/* Save result of I/O IN instr. in AL/AX/EAX. */
8489	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
8490	}
8491	else if (rc == VINF_IOM_R3_IOPORT_READ)
8492	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
8493	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
8494	}
8495	}
8496
8497	if (IOM_SUCCESS(rc))
8498	{
8499	if (!fUpdateRipAlready)
8500	{
8501	pMixedCtx->rip += cbInstr;
8502	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8503	}
8504	if (RT_LIKELY(rc == VINF_SUCCESS))
8505	{
8506	/*
8507	* If any I/O breakpoints are armed, then we should check if a
8508	* debug trap needs to be generated.
8509	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
8510	*/
8511	/** @todo We're not honoring I/O BPs if informational status code is returned.
8512	* We're also ignoring our own debugger's attempt at using I/O
8513	* breakpoints. The whole host & guest debugger stuff needs to be
8514	* looked over at some point. For now, it's just best effort. */
8515	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
8516	AssertRCReturn(rc, rc);
8517	uint32_t const uDr7 = pMixedCtx->dr[7];
8518	if ( (uDr7 & X86_DR7_ENABLED_MASK)
8519	&& X86_DR7_ANY_RW_IO(uDr7)
8520	&& (pMixedCtx->cr4 & X86_CR4_DE) )
8521	{
8522	/** @todo We're a little late here if we're doing string I/O, as we're supposed
8523	* to break after the each repetition. Not sooo important, just for a
8524	* rainy day. (Should probably refactor some of this code; after the uDr7
8525	* detection let someone else handle it.) */
8526	/** @todo The AMD is mumbling something that sounds like cbValue == cbBp. The
8527	* Intel manual describes it differently, data and I/O breakpoints are to
8528	* be matched in the same way, probably. Bochs does it that way. We've
8529	* implemented it that way too, but it would be worth having a
8530	* bootsector testcase for asserting the correct behavior (as well as
8531	* correctness of this code). */
8532	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
8533	uint32_t uIOPortLast = uIOPort + cbValue - 1;
8534	for (unsigned iBp = 0; iBp < 4; iBp++)
8535	{
8536	if ( (uDr7 & X86_DR7_L_G(iBp))
8537	&& X86_DR7_GET_RW(uDr7, iBp) == X86_DR7_RW_IO)
8538	{
8539	/* ASSUME the breakpoint and the I/O width qualifier uses the same encoding (1 2 x 4). */
8540	static uint8_t const s_abInvAlign[4] = { 0, 1, 7, 3 };
8541	uint8_t cbInvAlign = s_abInvAlign[X86_DR7_GET_LEN(uDr7, iBp)];
8542	uint64_t uDrXFirst = pMixedCtx->dr[iBp] & ~(uint64_t)cbInvAlign;
8543	uint64_t uDrXLast = uDrXFirst + cbInvAlign;
8544	if (uDrXFirst <= uIOPortLast && uDrXLast >= uIOPort)
8545	{
8546	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8547	uint64_t uDR6 = ASMGetDR6();
8548
8549	/* Clear all breakpoint status flags and set the one we just hit. */
8550	uDR6 &= ~X86_DR6_B_MASK;
8551	uDR6 \|= X86_DR6_B(iBp);
8552
8553	/*
8554	* Note: AMD64 Architecture Programmer's Manual 13.1:
8555	* Bits 15:13 of the DR6 register is never cleared by the processor and must
8556	* be cleared by software after the contents have been read.
8557	*/
8558	ASMSetDR6(uDR6);
8559
8560	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8561	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8562
8563	/* Paranoia. */
8564	pMixedCtx->dr[7] &= ~(X86_DR7_RAZ_MASK \| X86_DR7_MBZ_MASK);
8565	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
8566
8567	/* Resync DR7 */
8568	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
8569	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8570
8571	/* Set #DB to be injected into the VM and continue guest execution. */
8572	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
8573	break;
8574	}
8575	}
8576	}
8577	}
8578	}
8579	}
8580
8581	#ifdef DEBUG
8582	if (rc == VINF_IOM_R3_IOPORT_READ)
8583	Assert(!fIOWrite);
8584	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
8585	Assert(fIOWrite);
8586	else
8587	{
8588	AssertMsg( RT_FAILURE(rc)
8589	\|\| rc == VINF_SUCCESS
8590	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
8591	\|\| rc == VINF_EM_RAW_GUEST_TRAP
8592	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
8593	}
8594	#endif
8595
8596	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
8597	return rc;
8598	}
8599
8600
8601	/**
8602	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
8603	* VM-exit.
8604	*/
8605	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8606	{
8607	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8608
8609	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
8610	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8611	AssertRCReturn(rc, rc);
8612	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
8613	{
8614	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
8615	AssertRCReturn(rc, rc);
8616	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
8617	{
8618	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
8619
8620	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
8621	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
8622	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
8623	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
8624	{
8625	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
8626	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
8627
8628	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
8629	Assert(!pVCpu->hm.s.Event.fPending);
8630	pVCpu->hm.s.Event.fPending = true;
8631	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
8632	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
8633	AssertRCReturn(rc, rc);
8634	if (fErrorCodeValid)
8635	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
8636	else
8637	pVCpu->hm.s.Event.u32ErrCode = 0;
8638	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8639	&& uVector == X86_XCPT_PF)
8640	{
8641	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
8642	}
8643
8644	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
8645	}
8646	}
8647	}
8648
8649	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8650	* emulation. */
8651	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8652	return VERR_EM_INTERPRETER;
8653	}
8654
8655
8656	/**
8657	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
8658	*/
8659	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8660	{
8661	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8662	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
8663	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
8664	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8665	AssertRCReturn(rc, rc);
8666	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
8667	return VINF_EM_DBG_STOP;
8668	}
8669
8670
8671	/**
8672	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
8673	*/
8674	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8675	{
8676	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8677
8678	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8679	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8680	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8681	return VINF_SUCCESS;
8682	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8683	return rc;
8684
8685	#if 0
8686	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
8687	* just sync the whole thing. */
8688	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8689	#else
8690	/* Aggressive state sync. for now. */
8691	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8692	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8693	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8694	#endif
8695	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8696	AssertRCReturn(rc, rc);
8697
8698	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8699	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
8700	switch (uAccessType)
8701	{
8702	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8703	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8704	{
8705	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8706	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
8707	{
8708	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
8709	}
8710
8711	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
8712	GCPhys &= PAGE_BASE_GC_MASK;
8713	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
8714	PVM pVM = pVCpu->CTX_SUFF(pVM);
8715	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
8716	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
8717
8718	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
8719	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
8720	CPUMCTX2CORE(pMixedCtx), GCPhys);
8721	rc = VBOXSTRICTRC_VAL(rc2);
8722	Log4(("ApicAccess rc=%d\n", rc));
8723	if ( rc == VINF_SUCCESS
8724	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8725	\|\| rc == VERR_PAGE_NOT_PRESENT)
8726	{
8727	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8728	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8729	rc = VINF_SUCCESS;
8730	}
8731	break;
8732	}
8733
8734	default:
8735	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
8736	rc = VINF_EM_RAW_EMULATE_INSTR;
8737	break;
8738	}
8739
8740	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
8741	return rc;
8742	}
8743
8744
8745	/**
8746	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8747	* VM-exit.
8748	*/
8749	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8750	{
8751	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8752
8753	/* We should -not- get this VM-exit if the guest is debugging. */
8754	if (CPUMIsGuestDebugStateActive(pVCpu))
8755	{
8756	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8757	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8758	}
8759
8760	int rc = VERR_INTERNAL_ERROR_5;
8761	if ( !DBGFIsStepping(pVCpu)
8762	&& !CPUMIsHyperDebugStateActive(pVCpu))
8763	{
8764	/* Don't intercept MOV DRx. */
8765	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
8766	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8767	AssertRCReturn(rc, rc);
8768
8769	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8770	PVM pVM = pVCpu->CTX_SUFF(pVM);
8771	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
8772	AssertRC(rc);
8773	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8774
8775	#ifdef VBOX_WITH_STATISTICS
8776	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8777	AssertRCReturn(rc, rc);
8778	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8779	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8780	else
8781	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8782	#endif
8783	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
8784	return VINF_SUCCESS;
8785	}
8786
8787	/*
8788	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
8789	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
8790	*/
8791	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8792	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8793	AssertRCReturn(rc, rc);
8794
8795	PVM pVM = pVCpu->CTX_SUFF(pVM);
8796	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8797	{
8798	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8799	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
8800	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
8801	if (RT_SUCCESS(rc))
8802	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8803	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8804	}
8805	else
8806	{
8807	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8808	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
8809	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
8810	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8811	}
8812
8813	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8814	if (RT_SUCCESS(rc))
8815	{
8816	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8817	AssertRCReturn(rc2, rc2);
8818	}
8819	return rc;
8820	}
8821
8822
8823	/**
8824	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8825	* Conditional VM-exit.
8826	*/
8827	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8828	{
8829	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8830	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8831
8832	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8833	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8834	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8835	return VINF_SUCCESS;
8836	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8837	return rc;
8838
8839	RTGCPHYS GCPhys = 0;
8840	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8841
8842	#if 0
8843	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8844	#else
8845	/* Aggressive state sync. for now. */
8846	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8847	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8848	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8849	#endif
8850	AssertRCReturn(rc, rc);
8851
8852	/*
8853	* If we succeed, resume guest execution.
8854	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8855	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8856	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8857	* weird case. See @bugref{6043}.
8858	*/
8859	PVM pVM = pVCpu->CTX_SUFF(pVM);
8860	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8861	rc = VBOXSTRICTRC_VAL(rc2);
8862	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8863	if ( rc == VINF_SUCCESS
8864	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8865	\|\| rc == VERR_PAGE_NOT_PRESENT)
8866	{
8867	/* Successfully handled MMIO operation. */
8868	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8869	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8870	rc = VINF_SUCCESS;
8871	}
8872	return rc;
8873	}
8874
8875
8876	/**
8877	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8878	* VM-exit.
8879	*/
8880	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8881	{
8882	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8883	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8884
8885	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8886	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8887	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8888	return VINF_SUCCESS;
8889	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8890	return rc;
8891
8892	RTGCPHYS GCPhys = 0;
8893	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8894	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8895	#if 0
8896	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8897	#else
8898	/* Aggressive state sync. for now. */
8899	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8900	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8901	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8902	#endif
8903	AssertRCReturn(rc, rc);
8904
8905	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8906	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
8907
8908	RTGCUINT uErrorCode = 0;
8909	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8910	uErrorCode \|= X86_TRAP_PF_ID;
8911	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8912	uErrorCode \|= X86_TRAP_PF_RW;
8913	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8914	uErrorCode \|= X86_TRAP_PF_P;
8915
8916	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
8917
8918	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
8919	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8920
8921	/* Handle the pagefault trap for the nested shadow table. */
8922	PVM pVM = pVCpu->CTX_SUFF(pVM);
8923	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8924	TRPMResetTrap(pVCpu);
8925
8926	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8927	if ( rc == VINF_SUCCESS
8928	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8929	\|\| rc == VERR_PAGE_NOT_PRESENT)
8930	{
8931	/* Successfully synced our nested page tables. */
8932	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8933	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8934	return VINF_SUCCESS;
8935	}
8936
8937	Log4(("EPT return to ring-3 rc=%d\n"));
8938	return rc;
8939	}
8940
8941	/** @} */
8942
8943	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8944	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8945	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8946
8947	/** @name VM-exit exception handlers.
8948	* @{
8949	*/
8950
8951	/**
8952	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8953	*/
8954	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8955	{
8956	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8957	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8958
8959	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8960	AssertRCReturn(rc, rc);
8961
8962	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8963	{
8964	/* Old-style FPU error reporting needs some extra work. */
8965	/** @todo don't fall back to the recompiler, but do it manually. */
8966	return VERR_EM_INTERPRETER;
8967	}
8968
8969	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8970	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8971	return rc;
8972	}
8973
8974
8975	/**
8976	* VM-exit exception handler for #BP (Breakpoint exception).
8977	*/
8978	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8979	{
8980	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8981	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8982
8983	/** @todo Try optimize this by not saving the entire guest state unless
8984	* really needed. */
8985	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8986	AssertRCReturn(rc, rc);
8987
8988	PVM pVM = pVCpu->CTX_SUFF(pVM);
8989	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8990	if (rc == VINF_EM_RAW_GUEST_TRAP)
8991	{
8992	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8993	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8994	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8995	AssertRCReturn(rc, rc);
8996
8997	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8998	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8999	}
9000
9001	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
9002	return rc;
9003	}
9004
9005
9006	/**
9007	* VM-exit exception handler for #DB (Debug exception).
9008	*/
9009	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9010	{
9011	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9012	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
9013
9014	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9015	AssertRCReturn(rc, rc);
9016
9017	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
9018	uint64_t uDR6 = X86_DR6_INIT_VAL;
9019	uDR6 \|= (pVmxTransient->uExitQualification
9020	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
9021	PVM pVM = pVCpu->CTX_SUFF(pVM);
9022	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
9023	if (rc == VINF_EM_RAW_GUEST_TRAP)
9024	{
9025	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
9026	pMixedCtx->dr[6] = uDR6;
9027
9028	if (CPUMIsGuestDebugStateActive(pVCpu))
9029	ASMSetDR6(pMixedCtx->dr[6]);
9030
9031	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
9032
9033	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
9034	pMixedCtx->dr[7] &= ~X86_DR7_GD;
9035
9036	/* Paranoia. */
9037	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits MBZ. */
9038	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
9039	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
9040
9041	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
9042	AssertRCReturn(rc,rc);
9043
9044	int rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9045	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9046	rc2 \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
9047	AssertRCReturn(rc2, rc2);
9048	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9049	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
9050	rc = VINF_SUCCESS;
9051	}
9052
9053	return rc;
9054	}
9055
9056
9057	/**
9058	* VM-exit exception handler for #NM (Device-not-available exception: floating
9059	* point exception).
9060	*/
9061	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9062	{
9063	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9064
9065	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9066	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
9067	#endif
9068
9069	/* We require CR0 and EFER. EFER is always up-to-date. */
9070	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9071	AssertRCReturn(rc, rc);
9072
9073	/* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
9074	PVM pVM = pVCpu->CTX_SUFF(pVM);
9075	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
9076	if (rc == VINF_SUCCESS)
9077	{
9078	Assert(CPUMIsGuestFPUStateActive(pVCpu));
9079	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
9080	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
9081	return VINF_SUCCESS;
9082	}
9083
9084	/* Forward #NM to the guest. */
9085	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
9086	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9087	AssertRCReturn(rc, rc);
9088	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9089	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
9090	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
9091	return rc;
9092	}
9093
9094
9095	/**
9096	* VM-exit exception handler for #GP (General-protection exception).
9097	*
9098	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
9099	*/
9100	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9101	{
9102	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9103	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
9104
9105	int rc = VERR_INTERNAL_ERROR_5;
9106	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9107	{
9108	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9109	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
9110	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9111	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
9112	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9113	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9114	AssertRCReturn(rc, rc);
9115	Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
9116	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
9117	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9118	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
9119	return rc;
9120	#else
9121	/* We don't intercept #GP. */
9122	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
9123	return VERR_VMX_UNEXPECTED_EXCEPTION;
9124	#endif
9125	}
9126
9127	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9128	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
9129
9130	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
9131	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9132	AssertRCReturn(rc, rc);
9133
9134	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
9135	uint32_t cbOp = 0;
9136	PVM pVM = pVCpu->CTX_SUFF(pVM);
9137	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
9138	if (RT_SUCCESS(rc))
9139	{
9140	rc = VINF_SUCCESS;
9141	Assert(cbOp == pDis->cbInstr);
9142	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
9143	switch (pDis->pCurInstr->uOpcode)
9144	{
9145	case OP_CLI:
9146	{
9147	pMixedCtx->eflags.Bits.u1IF = 0;
9148	pMixedCtx->rip += pDis->cbInstr;
9149	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
9150	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
9151	break;
9152	}
9153
9154	case OP_STI:
9155	{
9156	pMixedCtx->eflags.Bits.u1IF = 1;
9157	pMixedCtx->rip += pDis->cbInstr;
9158	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
9159	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
9160	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
9161	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
9162	break;
9163	}
9164
9165	case OP_HLT:
9166	{
9167	rc = VINF_EM_HALT;
9168	pMixedCtx->rip += pDis->cbInstr;
9169	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
9170	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9171	break;
9172	}
9173
9174	case OP_POPF:
9175	{
9176	Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
9177	uint32_t cbParm = 0;
9178	uint32_t uMask = 0;
9179	if (pDis->fPrefix & DISPREFIX_OPSIZE)
9180	{
9181	cbParm = 4;
9182	uMask = 0xffffffff;
9183	}
9184	else
9185	{
9186	cbParm = 2;
9187	uMask = 0xffff;
9188	}
9189
9190	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
9191	RTGCPTR GCPtrStack = 0;
9192	X86EFLAGS uEflags;
9193	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
9194	&GCPtrStack);
9195	if (RT_SUCCESS(rc))
9196	{
9197	Assert(sizeof(uEflags.u32) >= cbParm);
9198	uEflags.u32 = 0;
9199	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
9200	}
9201	if (RT_FAILURE(rc))
9202	{
9203	rc = VERR_EM_INTERPRETER;
9204	break;
9205	}
9206	Log4(("POPF %x -> %#RX64 mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
9207	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
9208	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
9209	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
9210	pMixedCtx->eflags.Bits.u1RF = 0;
9211	pMixedCtx->esp += cbParm;
9212	pMixedCtx->esp &= uMask;
9213	pMixedCtx->rip += pDis->cbInstr;
9214	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
9215	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
9216	break;
9217	}
9218
9219	case OP_PUSHF:
9220	{
9221	uint32_t cbParm = 0;
9222	uint32_t uMask = 0;
9223	if (pDis->fPrefix & DISPREFIX_OPSIZE)
9224	{
9225	cbParm = 4;
9226	uMask = 0xffffffff;
9227	}
9228	else
9229	{
9230	cbParm = 2;
9231	uMask = 0xffff;
9232	}
9233
9234	/* Get the stack pointer & push the contents of eflags onto the stack. */
9235	RTGCPTR GCPtrStack = 0;
9236	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
9237	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
9238	if (RT_FAILURE(rc))
9239	{
9240	rc = VERR_EM_INTERPRETER;
9241	break;
9242	}
9243	X86EFLAGS uEflags;
9244	uEflags = pMixedCtx->eflags;
9245	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
9246	uEflags.Bits.u1RF = 0;
9247	uEflags.Bits.u1VM = 0;
9248
9249	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
9250	if (RT_FAILURE(rc))
9251	{
9252	rc = VERR_EM_INTERPRETER;
9253	break;
9254	}
9255	Log4(("PUSHF %x -> %#RGv\n", uEflags.u, GCPtrStack));
9256	pMixedCtx->esp -= cbParm;
9257	pMixedCtx->esp &= uMask;
9258	pMixedCtx->rip += pDis->cbInstr;
9259	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
9260	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
9261	break;
9262	}
9263
9264	case OP_IRET:
9265	{
9266	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
9267	* instruction reference. */
9268	RTGCPTR GCPtrStack = 0;
9269	uint32_t uMask = 0xffff;
9270	uint16_t aIretFrame[3];
9271	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
9272	{
9273	rc = VERR_EM_INTERPRETER;
9274	break;
9275	}
9276	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
9277	&GCPtrStack);
9278	if (RT_SUCCESS(rc))
9279	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
9280	if (RT_FAILURE(rc))
9281	{
9282	rc = VERR_EM_INTERPRETER;
9283	break;
9284	}
9285	pMixedCtx->eip = 0;
9286	pMixedCtx->ip = aIretFrame[0];
9287	pMixedCtx->cs.Sel = aIretFrame[1];
9288	pMixedCtx->cs.ValidSel = aIretFrame[1];
9289	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
9290	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
9291	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
9292	pMixedCtx->sp += sizeof(aIretFrame);
9293	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
9294	\| HM_CHANGED_GUEST_RFLAGS;
9295	Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
9296	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
9297	break;
9298	}
9299
9300	case OP_INT:
9301	{
9302	uint16_t uVector = pDis->Param1.uValue & 0xff;
9303	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
9304	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
9305	break;
9306	}
9307
9308	case OP_INTO:
9309	{
9310	if (pMixedCtx->eflags.Bits.u1OF)
9311	{
9312	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
9313	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
9314	}
9315	break;
9316	}
9317
9318	default:
9319	{
9320	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
9321	EMCODETYPE_SUPERVISOR);
9322	rc = VBOXSTRICTRC_VAL(rc2);
9323	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
9324	Log4(("#GP rc=%Rrc\n", rc));
9325	break;
9326	}
9327	}
9328	}
9329	else
9330	rc = VERR_EM_INTERPRETER;
9331
9332	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
9333	("#GP Unexpected rc=%Rrc\n", rc));
9334	return rc;
9335	}
9336
9337
9338	/**
9339	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
9340	* the exception reported in the VMX transient structure back into the VM.
9341	*
9342	* @remarks Requires uExitIntrInfo in the VMX transient structure to be
9343	* up-to-date.
9344	*/
9345	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9346	{
9347	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9348
9349	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
9350	hmR0VmxCheckExitDueToEventDelivery(). */
9351	int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
9352	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9353	AssertRCReturn(rc, rc);
9354	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
9355
9356	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9357	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
9358	return VINF_SUCCESS;
9359	}
9360
9361
9362	/**
9363	* VM-exit exception handler for #PF (Page-fault exception).
9364	*/
9365	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9366	{
9367	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9368	PVM pVM = pVCpu->CTX_SUFF(pVM);
9369	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9370	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9371	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
9372	AssertRCReturn(rc, rc);
9373
9374	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
9375	if (pVM->hm.s.fNestedPaging)
9376	{
9377	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
9378	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
9379	{
9380	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
9381	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9382	0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
9383	}
9384	else
9385	{
9386	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
9387	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
9388	Log4(("Pending #DF due to vectoring #PF. NP\n"));
9389	}
9390	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
9391	return rc;
9392	}
9393	#else
9394	Assert(!pVM->hm.s.fNestedPaging);
9395	#endif
9396
9397	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9398	AssertRCReturn(rc, rc);
9399
9400	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
9401	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
9402
9403	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
9404	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
9405	(RTGCPTR)pVmxTransient->uExitQualification);
9406
9407	Log4(("#PF: rc=%Rrc\n", rc));
9408	if (rc == VINF_SUCCESS)
9409	{
9410	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
9411	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
9412	* memory? We don't update the whole state here... */
9413	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
9414	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
9415	TRPMResetTrap(pVCpu);
9416	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
9417	return rc;
9418	}
9419	else if (rc == VINF_EM_RAW_GUEST_TRAP)
9420	{
9421	if (!pVmxTransient->fVectoringPF)
9422	{
9423	/* It's a guest page fault and needs to be reflected to the guest. */
9424	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
9425	TRPMResetTrap(pVCpu);
9426	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
9427	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
9428	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9429	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
9430	}
9431	else
9432	{
9433	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
9434	TRPMResetTrap(pVCpu);
9435	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
9436	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
9437	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
9438	}
9439
9440	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
9441	return VINF_SUCCESS;
9442	}
9443
9444	TRPMResetTrap(pVCpu);
9445	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
9446	return rc;
9447	}
9448
9449	/** @} */
9450

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 47565

Download in other formats: