HMSVMR0.cpp@ 95248

Last change on this file since 95248 was 95048, checked in by vboxsync, 3 years ago
GCM: Corrected return codes.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 381.8 KB

Line
1	/* $Id: HMSVMR0.cpp 95048 2022-05-20 07:28:15Z vboxsync $ */
2	/** @file
3	* HM SVM (AMD-V) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2013-2022 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/asm-amd64-x86.h>
25	#include <iprt/thread.h>
26
27	#include <VBox/vmm/pdmapi.h>
28	#include <VBox/vmm/dbgf.h>
29	#include <VBox/vmm/iem.h>
30	#include <VBox/vmm/iom.h>
31	#include <VBox/vmm/tm.h>
32	#include <VBox/vmm/em.h>
33	#include <VBox/vmm/gcm.h>
34	#include <VBox/vmm/gim.h>
35	#include <VBox/vmm/apic.h>
36	#include "HMInternal.h"
37	#include <VBox/vmm/vmcc.h>
38	#include <VBox/err.h>
39	#include "HMSVMR0.h"
40	#include "dtrace/VBoxVMM.h"
41
42	#ifdef DEBUG_ramshankar
43	# define HMSVM_SYNC_FULL_GUEST_STATE
44	# define HMSVM_ALWAYS_TRAP_ALL_XCPTS
45	# define HMSVM_ALWAYS_TRAP_PF
46	# define HMSVM_ALWAYS_TRAP_TASK_SWITCH
47	#endif
48
49
50	/*********************************************************************************************************************************
51	* Defined Constants And Macros *
52	*********************************************************************************************************************************/
53	#ifdef VBOX_WITH_STATISTICS
54	# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
55	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll); \
56	if ((u64ExitCode) == SVM_EXIT_NPF) \
57	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf); \
58	else \
59	STAM_COUNTER_INC(&pVCpu->hm.s.aStatExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
60	} while (0)
61
62	# define HMSVM_DEBUG_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
63	STAM_COUNTER_INC(&pVCpu->hm.s.StatDebugExitAll); \
64	if ((u64ExitCode) == SVM_EXIT_NPF) \
65	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf); \
66	else \
67	STAM_COUNTER_INC(&pVCpu->hm.s.aStatExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
68	} while (0)
69
70	# define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
71	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitAll); \
72	if ((u64ExitCode) == SVM_EXIT_NPF) \
73	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitReasonNpf); \
74	else \
75	STAM_COUNTER_INC(&pVCpu->hm.s.aStatNestedExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
76	} while (0)
77	#else
78	# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
79	# define HMSVM_DEBUG_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
80	# define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
81	#endif /* !VBOX_WITH_STATISTICS */
82
83	/** If we decide to use a function table approach this can be useful to
84	* switch to a "static DECLCALLBACK(int)". */
85	#define HMSVM_EXIT_DECL static VBOXSTRICTRC
86
87	/**
88	* Subset of the guest-CPU state that is kept by SVM R0 code while executing the
89	* guest using hardware-assisted SVM.
90	*
91	* This excludes state like TSC AUX, GPRs (other than RSP, RAX) which are always
92	* are swapped and restored across the world-switch and also registers like
93	* EFER, PAT MSR etc. which cannot be modified by the guest without causing a
94	* \#VMEXIT.
95	*/
96	#define HMSVM_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
97	\| CPUMCTX_EXTRN_RFLAGS \
98	\| CPUMCTX_EXTRN_RAX \
99	\| CPUMCTX_EXTRN_RSP \
100	\| CPUMCTX_EXTRN_SREG_MASK \
101	\| CPUMCTX_EXTRN_CR0 \
102	\| CPUMCTX_EXTRN_CR2 \
103	\| CPUMCTX_EXTRN_CR3 \
104	\| CPUMCTX_EXTRN_TABLE_MASK \
105	\| CPUMCTX_EXTRN_DR6 \
106	\| CPUMCTX_EXTRN_DR7 \
107	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
108	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
109	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
110	\| CPUMCTX_EXTRN_HWVIRT \
111	\| CPUMCTX_EXTRN_INHIBIT_INT \
112	\| CPUMCTX_EXTRN_HM_SVM_MASK)
113
114	/**
115	* Subset of the guest-CPU state that is shared between the guest and host.
116	*/
117	#define HMSVM_CPUMCTX_SHARED_STATE CPUMCTX_EXTRN_DR_MASK
118
119	/** Macro for importing guest state from the VMCB back into CPUMCTX. */
120	#define HMSVM_CPUMCTX_IMPORT_STATE(a_pVCpu, a_fWhat) \
121	do { \
122	if ((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fWhat)) \
123	hmR0SvmImportGuestState((a_pVCpu), (a_fWhat)); \
124	} while (0)
125
126	/** Assert that the required state bits are fetched. */
127	#define HMSVM_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
128	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
129	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
130
131	/** Assert that preemption is disabled or covered by thread-context hooks. */
132	#define HMSVM_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
133	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
134
135	/** Assert that we haven't migrated CPUs when thread-context hooks are not
136	* used. */
137	#define HMSVM_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
138	\|\| (a_pVCpu)->hmr0.s.idEnteredCpu == RTMpCpuId(), \
139	("Illegal migration! Entered on CPU %u Current %u\n", \
140	(a_pVCpu)->hmr0.s.idEnteredCpu, RTMpCpuId()));
141
142	/** Assert that we're not executing a nested-guest. */
143	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
144	# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) Assert(!CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
145	#else
146	# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
147	#endif
148
149	/** Assert that we're executing a nested-guest. */
150	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
151	# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) Assert(CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
152	#else
153	# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
154	#endif
155
156	/** Macro for checking and returning from the using function for
157	* \#VMEXIT intercepts that maybe caused during delivering of another
158	* event in the guest. */
159	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
160	# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(a_pVCpu, a_pSvmTransient) \
161	do \
162	{ \
163	int rc = hmR0SvmCheckExitDueToEventDelivery((a_pVCpu), (a_pSvmTransient)); \
164	if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
165	else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
166	else if ( rc == VINF_EM_RESET \
167	&& CPUMIsGuestSvmCtrlInterceptSet((a_pVCpu), &(a_pVCpu)->cpum.GstCtx, SVM_CTRL_INTERCEPT_SHUTDOWN)) \
168	{ \
169	HMSVM_CPUMCTX_IMPORT_STATE((a_pVCpu), HMSVM_CPUMCTX_EXTRN_ALL); \
170	return IEMExecSvmVmexit((a_pVCpu), SVM_EXIT_SHUTDOWN, 0, 0); \
171	} \
172	else \
173	return rc; \
174	} while (0)
175	#else
176	# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(a_pVCpu, a_pSvmTransient) \
177	do \
178	{ \
179	int rc = hmR0SvmCheckExitDueToEventDelivery((a_pVCpu), (a_pSvmTransient)); \
180	if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
181	else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
182	else \
183	return rc; \
184	} while (0)
185	#endif
186
187	/** Macro for upgrading a @a a_rc to VINF_EM_DBG_STEPPED after emulating an
188	* instruction that exited. */
189	#define HMSVM_CHECK_SINGLE_STEP(a_pVCpu, a_rc) \
190	do { \
191	if ((a_pVCpu)->hm.s.fSingleInstruction && (a_rc) == VINF_SUCCESS) \
192	(a_rc) = VINF_EM_DBG_STEPPED; \
193	} while (0)
194
195	/** Validate segment descriptor granularity bit. */
196	#ifdef VBOX_STRICT
197	# define HMSVM_ASSERT_SEG_GRANULARITY(a_pCtx, reg) \
198	AssertMsg( !(a_pCtx)->reg.Attr.n.u1Present \
199	\|\| ( (a_pCtx)->reg.Attr.n.u1Granularity \
200	? ((a_pCtx)->reg.u32Limit & 0xfff) == 0xfff \
201	: (a_pCtx)->reg.u32Limit <= UINT32_C(0xfffff)), \
202	("Invalid Segment Attributes Limit=%#RX32 Attr=%#RX32 Base=%#RX64\n", (a_pCtx)->reg.u32Limit, \
203	(a_pCtx)->reg.Attr.u, (a_pCtx)->reg.u64Base))
204	#else
205	# define HMSVM_ASSERT_SEG_GRANULARITY(a_pCtx, reg) do { } while (0)
206	#endif
207
208	/**
209	* Exception bitmap mask for all contributory exceptions.
210	*
211	* Page fault is deliberately excluded here as it's conditional as to whether
212	* it's contributory or benign. Page faults are handled separately.
213	*/
214	#define HMSVM_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
215	\| RT_BIT(X86_XCPT_DE))
216
217	/**
218	* Mandatory/unconditional guest control intercepts.
219	*
220	* SMIs can and do happen in normal operation. We need not intercept them
221	* while executing the guest (or nested-guest).
222	*/
223	#define HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS ( SVM_CTRL_INTERCEPT_INTR \
224	\| SVM_CTRL_INTERCEPT_NMI \
225	\| SVM_CTRL_INTERCEPT_INIT \
226	\| SVM_CTRL_INTERCEPT_RDPMC \
227	\| SVM_CTRL_INTERCEPT_CPUID \
228	\| SVM_CTRL_INTERCEPT_RSM \
229	\| SVM_CTRL_INTERCEPT_HLT \
230	\| SVM_CTRL_INTERCEPT_IOIO_PROT \
231	\| SVM_CTRL_INTERCEPT_MSR_PROT \
232	\| SVM_CTRL_INTERCEPT_INVLPGA \
233	\| SVM_CTRL_INTERCEPT_SHUTDOWN \
234	\| SVM_CTRL_INTERCEPT_FERR_FREEZE \
235	\| SVM_CTRL_INTERCEPT_VMRUN \
236	\| SVM_CTRL_INTERCEPT_SKINIT \
237	\| SVM_CTRL_INTERCEPT_WBINVD \
238	\| SVM_CTRL_INTERCEPT_MONITOR \
239	\| SVM_CTRL_INTERCEPT_MWAIT \
240	\| SVM_CTRL_INTERCEPT_CR0_SEL_WRITE \
241	\| SVM_CTRL_INTERCEPT_XSETBV)
242
243	/** @name VMCB Clean Bits.
244	*
245	* These flags are used for VMCB-state caching. A set VMCB Clean bit indicates
246	* AMD-V doesn't need to reload the corresponding value(s) from the VMCB in
247	* memory.
248	*
249	* @{ */
250	/** All intercepts vectors, TSC offset, PAUSE filter counter. */
251	#define HMSVM_VMCB_CLEAN_INTERCEPTS RT_BIT(0)
252	/** I/O permission bitmap, MSR permission bitmap. */
253	#define HMSVM_VMCB_CLEAN_IOPM_MSRPM RT_BIT(1)
254	/** ASID. */
255	#define HMSVM_VMCB_CLEAN_ASID RT_BIT(2)
256	/** TRP: V_TPR, V_IRQ, V_INTR_PRIO, V_IGN_TPR, V_INTR_MASKING,
257	V_INTR_VECTOR. */
258	#define HMSVM_VMCB_CLEAN_INT_CTRL RT_BIT(3)
259	/** Nested Paging: Nested CR3 (nCR3), PAT. */
260	#define HMSVM_VMCB_CLEAN_NP RT_BIT(4)
261	/** Control registers (CR0, CR3, CR4, EFER). */
262	#define HMSVM_VMCB_CLEAN_CRX_EFER RT_BIT(5)
263	/** Debug registers (DR6, DR7). */
264	#define HMSVM_VMCB_CLEAN_DRX RT_BIT(6)
265	/** GDT, IDT limit and base. */
266	#define HMSVM_VMCB_CLEAN_DT RT_BIT(7)
267	/** Segment register: CS, SS, DS, ES limit and base. */
268	#define HMSVM_VMCB_CLEAN_SEG RT_BIT(8)
269	/** CR2.*/
270	#define HMSVM_VMCB_CLEAN_CR2 RT_BIT(9)
271	/** Last-branch record (DbgCtlMsr, br_from, br_to, lastint_from, lastint_to) */
272	#define HMSVM_VMCB_CLEAN_LBR RT_BIT(10)
273	/** AVIC (AVIC APIC_BAR; AVIC APIC_BACKING_PAGE, AVIC
274	PHYSICAL_TABLE and AVIC LOGICAL_TABLE Pointers). */
275	#define HMSVM_VMCB_CLEAN_AVIC RT_BIT(11)
276	/** Mask of all valid VMCB Clean bits. */
277	#define HMSVM_VMCB_CLEAN_ALL ( HMSVM_VMCB_CLEAN_INTERCEPTS \
278	\| HMSVM_VMCB_CLEAN_IOPM_MSRPM \
279	\| HMSVM_VMCB_CLEAN_ASID \
280	\| HMSVM_VMCB_CLEAN_INT_CTRL \
281	\| HMSVM_VMCB_CLEAN_NP \
282	\| HMSVM_VMCB_CLEAN_CRX_EFER \
283	\| HMSVM_VMCB_CLEAN_DRX \
284	\| HMSVM_VMCB_CLEAN_DT \
285	\| HMSVM_VMCB_CLEAN_SEG \
286	\| HMSVM_VMCB_CLEAN_CR2 \
287	\| HMSVM_VMCB_CLEAN_LBR \
288	\| HMSVM_VMCB_CLEAN_AVIC)
289	/** @} */
290
291	/**
292	* MSRPM (MSR permission bitmap) read permissions (for guest RDMSR).
293	*/
294	typedef enum SVMMSREXITREAD
295	{
296	/** Reading this MSR causes a \#VMEXIT. */
297	SVMMSREXIT_INTERCEPT_READ = 0xb,
298	/** Reading this MSR does not cause a \#VMEXIT. */
299	SVMMSREXIT_PASSTHRU_READ
300	} SVMMSREXITREAD;
301
302	/**
303	* MSRPM (MSR permission bitmap) write permissions (for guest WRMSR).
304	*/
305	typedef enum SVMMSREXITWRITE
306	{
307	/** Writing to this MSR causes a \#VMEXIT. */
308	SVMMSREXIT_INTERCEPT_WRITE = 0xd,
309	/** Writing to this MSR does not cause a \#VMEXIT. */
310	SVMMSREXIT_PASSTHRU_WRITE
311	} SVMMSREXITWRITE;
312
313	/**
314	* SVM \#VMEXIT handler.
315	*
316	* @returns Strict VBox status code.
317	* @param pVCpu The cross context virtual CPU structure.
318	* @param pSvmTransient Pointer to the SVM-transient structure.
319	*/
320	typedef VBOXSTRICTRC FNSVMEXITHANDLER(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
321
322
323	/*********************************************************************************************************************************
324	* Internal Functions *
325	*********************************************************************************************************************************/
326	static void hmR0SvmPendingEventToTrpmTrap(PVMCPUCC pVCpu);
327	static void hmR0SvmLeave(PVMCPUCC pVCpu, bool fImportState);
328
329
330	/** @name \#VMEXIT handlers.
331	* @{
332	*/
333	static FNSVMEXITHANDLER hmR0SvmExitIntr;
334	static FNSVMEXITHANDLER hmR0SvmExitWbinvd;
335	static FNSVMEXITHANDLER hmR0SvmExitInvd;
336	static FNSVMEXITHANDLER hmR0SvmExitCpuid;
337	static FNSVMEXITHANDLER hmR0SvmExitRdtsc;
338	static FNSVMEXITHANDLER hmR0SvmExitRdtscp;
339	static FNSVMEXITHANDLER hmR0SvmExitRdpmc;
340	static FNSVMEXITHANDLER hmR0SvmExitInvlpg;
341	static FNSVMEXITHANDLER hmR0SvmExitHlt;
342	static FNSVMEXITHANDLER hmR0SvmExitMonitor;
343	static FNSVMEXITHANDLER hmR0SvmExitMwait;
344	static FNSVMEXITHANDLER hmR0SvmExitShutdown;
345	static FNSVMEXITHANDLER hmR0SvmExitUnexpected;
346	static FNSVMEXITHANDLER hmR0SvmExitReadCRx;
347	static FNSVMEXITHANDLER hmR0SvmExitWriteCRx;
348	static FNSVMEXITHANDLER hmR0SvmExitMsr;
349	static FNSVMEXITHANDLER hmR0SvmExitReadDRx;
350	static FNSVMEXITHANDLER hmR0SvmExitWriteDRx;
351	static FNSVMEXITHANDLER hmR0SvmExitXsetbv;
352	static FNSVMEXITHANDLER hmR0SvmExitIOInstr;
353	static FNSVMEXITHANDLER hmR0SvmExitNestedPF;
354	static FNSVMEXITHANDLER hmR0SvmExitVIntr;
355	static FNSVMEXITHANDLER hmR0SvmExitTaskSwitch;
356	static FNSVMEXITHANDLER hmR0SvmExitVmmCall;
357	static FNSVMEXITHANDLER hmR0SvmExitPause;
358	static FNSVMEXITHANDLER hmR0SvmExitFerrFreeze;
359	static FNSVMEXITHANDLER hmR0SvmExitIret;
360	static FNSVMEXITHANDLER hmR0SvmExitXcptDE;
361	static FNSVMEXITHANDLER hmR0SvmExitXcptPF;
362	static FNSVMEXITHANDLER hmR0SvmExitXcptUD;
363	static FNSVMEXITHANDLER hmR0SvmExitXcptMF;
364	static FNSVMEXITHANDLER hmR0SvmExitXcptDB;
365	static FNSVMEXITHANDLER hmR0SvmExitXcptAC;
366	static FNSVMEXITHANDLER hmR0SvmExitXcptBP;
367	static FNSVMEXITHANDLER hmR0SvmExitXcptGP;
368	static FNSVMEXITHANDLER hmR0SvmExitXcptGeneric;
369	static FNSVMEXITHANDLER hmR0SvmExitSwInt;
370	static FNSVMEXITHANDLER hmR0SvmExitTrRead;
371	static FNSVMEXITHANDLER hmR0SvmExitTrWrite;
372	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
373	static FNSVMEXITHANDLER hmR0SvmExitClgi;
374	static FNSVMEXITHANDLER hmR0SvmExitStgi;
375	static FNSVMEXITHANDLER hmR0SvmExitVmload;
376	static FNSVMEXITHANDLER hmR0SvmExitVmsave;
377	static FNSVMEXITHANDLER hmR0SvmExitInvlpga;
378	static FNSVMEXITHANDLER hmR0SvmExitVmrun;
379	static FNSVMEXITHANDLER hmR0SvmNestedExitXcptDB;
380	static FNSVMEXITHANDLER hmR0SvmNestedExitXcptBP;
381	#endif
382	/** @} */
383
384	static VBOXSTRICTRC hmR0SvmHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
385	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
386	static VBOXSTRICTRC hmR0SvmHandleExitNested(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
387	#endif
388	static VBOXSTRICTRC hmR0SvmRunGuestCodeDebug(PVMCPUCC pVCpu, uint32_t *pcLoops);
389
390
391	/*********************************************************************************************************************************
392	* Global Variables *
393	*********************************************************************************************************************************/
394	/** Ring-0 memory object for the IO bitmap. */
395	static RTR0MEMOBJ g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
396	/** Physical address of the IO bitmap. */
397	static RTHCPHYS g_HCPhysIOBitmap;
398	/** Pointer to the IO bitmap. */
399	static R0PTRTYPE(void *) g_pvIOBitmap;
400
401	#ifdef VBOX_STRICT
402	# define HMSVM_LOG_RBP_RSP RT_BIT_32(0)
403	# define HMSVM_LOG_CR_REGS RT_BIT_32(1)
404	# define HMSVM_LOG_CS RT_BIT_32(2)
405	# define HMSVM_LOG_SS RT_BIT_32(3)
406	# define HMSVM_LOG_FS RT_BIT_32(4)
407	# define HMSVM_LOG_GS RT_BIT_32(5)
408	# define HMSVM_LOG_LBR RT_BIT_32(6)
409	# define HMSVM_LOG_ALL ( HMSVM_LOG_RBP_RSP \
410	\| HMSVM_LOG_CR_REGS \
411	\| HMSVM_LOG_CS \
412	\| HMSVM_LOG_SS \
413	\| HMSVM_LOG_FS \
414	\| HMSVM_LOG_GS \
415	\| HMSVM_LOG_LBR)
416
417	/**
418	* Dumps virtual CPU state and additional info. to the logger for diagnostics.
419	*
420	* @param pVCpu The cross context virtual CPU structure.
421	* @param pVmcb Pointer to the VM control block.
422	* @param pszPrefix Log prefix.
423	* @param fFlags Log flags, see HMSVM_LOG_XXX.
424	* @param uVerbose The verbosity level, currently unused.
425	*/
426	static void hmR0SvmLogState(PVMCPUCC pVCpu, PCSVMVMCB pVmcb, const char *pszPrefix, uint32_t fFlags, uint8_t uVerbose)
427	{
428	RT_NOREF2(pVCpu, uVerbose);
429	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
430
431	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
432	Log4(("%s: cs:rip=%04x:%RX64 efl=%#RX64\n", pszPrefix, pCtx->cs.Sel, pCtx->rip, pCtx->rflags.u));
433
434	if (fFlags & HMSVM_LOG_RBP_RSP)
435	{
436	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RBP);
437	Log4(("%s: rsp=%#RX64 rbp=%#RX64\n", pszPrefix, pCtx->rsp, pCtx->rbp));
438	}
439
440	if (fFlags & HMSVM_LOG_CR_REGS)
441	{
442	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4);
443	Log4(("%s: cr0=%#RX64 cr3=%#RX64 cr4=%#RX64\n", pszPrefix, pCtx->cr0, pCtx->cr3, pCtx->cr4));
444	}
445
446	if (fFlags & HMSVM_LOG_CS)
447	{
448	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
449	Log4(("%s: cs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->cs.Sel, pCtx->cs.u64Base,
450	pCtx->cs.u32Limit, pCtx->cs.Attr.u));
451	}
452	if (fFlags & HMSVM_LOG_SS)
453	{
454	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
455	Log4(("%s: ss={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->ss.Sel, pCtx->ss.u64Base,
456	pCtx->ss.u32Limit, pCtx->ss.Attr.u));
457	}
458	if (fFlags & HMSVM_LOG_FS)
459	{
460	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
461	Log4(("%s: fs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->fs.Sel, pCtx->fs.u64Base,
462	pCtx->fs.u32Limit, pCtx->fs.Attr.u));
463	}
464	if (fFlags & HMSVM_LOG_GS)
465	{
466	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
467	Log4(("%s: gs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->gs.Sel, pCtx->gs.u64Base,
468	pCtx->gs.u32Limit, pCtx->gs.Attr.u));
469	}
470
471	PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
472	if (fFlags & HMSVM_LOG_LBR)
473	{
474	Log4(("%s: br_from=%#RX64 br_to=%#RX64 lastxcpt_from=%#RX64 lastxcpt_to=%#RX64\n", pszPrefix, pVmcbGuest->u64BR_FROM,
475	pVmcbGuest->u64BR_TO, pVmcbGuest->u64LASTEXCPFROM, pVmcbGuest->u64LASTEXCPTO));
476	}
477	NOREF(pszPrefix); NOREF(pVmcbGuest); NOREF(pCtx);
478	}
479	#endif /* VBOX_STRICT */
480
481
482	/**
483	* Sets up and activates AMD-V on the current CPU.
484	*
485	* @returns VBox status code.
486	* @param pHostCpu The HM physical-CPU structure.
487	* @param pVM The cross context VM structure. Can be
488	* NULL after a resume!
489	* @param pvCpuPage Pointer to the global CPU page.
490	* @param HCPhysCpuPage Physical address of the global CPU page.
491	* @param fEnabledByHost Whether the host OS has already initialized AMD-V.
492	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs (currently
493	* unused).
494	*/
495	VMMR0DECL(int) SVMR0EnableCpu(PHMPHYSCPU pHostCpu, PVMCC pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
496	PCSUPHWVIRTMSRS pHwvirtMsrs)
497	{
498	Assert(!fEnabledByHost);
499	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
500	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
501	Assert(pvCpuPage); NOREF(pvCpuPage);
502	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
503
504	RT_NOREF2(fEnabledByHost, pHwvirtMsrs);
505
506	/* Paranoid: Disable interrupt as, in theory, interrupt handlers might mess with EFER. */
507	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
508
509	/*
510	* We must turn on AMD-V and setup the host state physical address, as those MSRs are per CPU.
511	*/
512	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
513	if (u64HostEfer & MSR_K6_EFER_SVME)
514	{
515	/* If the VBOX_HWVIRTEX_IGNORE_SVM_IN_USE is active, then we blindly use AMD-V. */
516	if ( pVM
517	&& pVM->hm.s.svm.fIgnoreInUseError)
518	pHostCpu->fIgnoreAMDVInUseError = true;
519
520	if (!pHostCpu->fIgnoreAMDVInUseError)
521	{
522	ASMSetFlags(fEFlags);
523	return VERR_SVM_IN_USE;
524	}
525	}
526
527	/* Turn on AMD-V in the EFER MSR. */
528	ASMWrMsr(MSR_K6_EFER, u64HostEfer \| MSR_K6_EFER_SVME);
529
530	/* Write the physical page address where the CPU will store the host state while executing the VM. */
531	ASMWrMsr(MSR_K8_VM_HSAVE_PA, HCPhysCpuPage);
532
533	/* Restore interrupts. */
534	ASMSetFlags(fEFlags);
535
536	/*
537	* Theoretically, other hypervisors may have used ASIDs, ideally we should flush all
538	* non-zero ASIDs when enabling SVM. AMD doesn't have an SVM instruction to flush all
539	* ASIDs (flushing is done upon VMRUN). Therefore, flag that we need to flush the TLB
540	* entirely with before executing any guest code.
541	*/
542	pHostCpu->fFlushAsidBeforeUse = true;
543
544	/*
545	* Ensure each VCPU scheduled on this CPU gets a new ASID on resume. See @bugref{6255}.
546	*/
547	++pHostCpu->cTlbFlushes;
548
549	return VINF_SUCCESS;
550	}
551
552
553	/**
554	* Deactivates AMD-V on the current CPU.
555	*
556	* @returns VBox status code.
557	* @param pHostCpu The HM physical-CPU structure.
558	* @param pvCpuPage Pointer to the global CPU page.
559	* @param HCPhysCpuPage Physical address of the global CPU page.
560	*/
561	VMMR0DECL(int) SVMR0DisableCpu(PHMPHYSCPU pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
562	{
563	RT_NOREF1(pHostCpu);
564	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
565	AssertReturn( HCPhysCpuPage
566	&& HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
567	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
568
569	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with EFER. */
570	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
571
572	/* Turn off AMD-V in the EFER MSR. */
573	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
574	ASMWrMsr(MSR_K6_EFER, u64HostEfer & ~MSR_K6_EFER_SVME);
575
576	/* Invalidate host state physical address. */
577	ASMWrMsr(MSR_K8_VM_HSAVE_PA, 0);
578
579	/* Restore interrupts. */
580	ASMSetFlags(fEFlags);
581
582	return VINF_SUCCESS;
583	}
584
585
586	/**
587	* Does global AMD-V initialization (called during module initialization).
588	*
589	* @returns VBox status code.
590	*/
591	VMMR0DECL(int) SVMR0GlobalInit(void)
592	{
593	/*
594	* Allocate 12 KB (3 pages) for the IO bitmap. Since this is non-optional and we always
595	* intercept all IO accesses, it's done once globally here instead of per-VM.
596	*/
597	Assert(g_hMemObjIOBitmap == NIL_RTR0MEMOBJ);
598	int rc = RTR0MemObjAllocCont(&g_hMemObjIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, false /* fExecutable */);
599	if (RT_FAILURE(rc))
600	return rc;
601
602	g_pvIOBitmap = RTR0MemObjAddress(g_hMemObjIOBitmap);
603	g_HCPhysIOBitmap = RTR0MemObjGetPagePhysAddr(g_hMemObjIOBitmap, 0 /* iPage */);
604
605	/* Set all bits to intercept all IO accesses. */
606	ASMMemFill32(g_pvIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
607
608	return VINF_SUCCESS;
609	}
610
611
612	/**
613	* Does global AMD-V termination (called during module termination).
614	*/
615	VMMR0DECL(void) SVMR0GlobalTerm(void)
616	{
617	if (g_hMemObjIOBitmap != NIL_RTR0MEMOBJ)
618	{
619	RTR0MemObjFree(g_hMemObjIOBitmap, true /* fFreeMappings */);
620	g_pvIOBitmap = NULL;
621	g_HCPhysIOBitmap = 0;
622	g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
623	}
624	}
625
626
627	/**
628	* Frees any allocated per-VCPU structures for a VM.
629	*
630	* @param pVM The cross context VM structure.
631	*/
632	DECLINLINE(void) hmR0SvmFreeStructs(PVMCC pVM)
633	{
634	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
635	{
636	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
637	AssertPtr(pVCpu);
638
639	if (pVCpu->hmr0.s.svm.hMemObjVmcbHost != NIL_RTR0MEMOBJ)
640	{
641	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjVmcbHost, false);
642	pVCpu->hmr0.s.svm.HCPhysVmcbHost = 0;
643	pVCpu->hmr0.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
644	}
645
646	if (pVCpu->hmr0.s.svm.hMemObjVmcb != NIL_RTR0MEMOBJ)
647	{
648	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjVmcb, false);
649	pVCpu->hmr0.s.svm.pVmcb = NULL;
650	pVCpu->hmr0.s.svm.HCPhysVmcb = 0;
651	pVCpu->hmr0.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
652	}
653
654	if (pVCpu->hmr0.s.svm.hMemObjMsrBitmap != NIL_RTR0MEMOBJ)
655	{
656	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjMsrBitmap, false);
657	pVCpu->hmr0.s.svm.pvMsrBitmap = NULL;
658	pVCpu->hmr0.s.svm.HCPhysMsrBitmap = 0;
659	pVCpu->hmr0.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
660	}
661	}
662	}
663
664
665	/**
666	* Sets pfnVMRun to the best suited variant.
667	*
668	* This must be called whenever anything changes relative to the SVMR0VMRun
669	* variant selection:
670	* - pVCpu->hm.s.fLoadSaveGuestXcr0
671	* - CPUMCTX_WSF_IBPB_ENTRY in pVCpu->cpum.GstCtx.fWorldSwitcher
672	* - CPUMCTX_WSF_IBPB_EXIT in pVCpu->cpum.GstCtx.fWorldSwitcher
673	* - Perhaps: CPUMIsGuestFPUStateActive() (windows only)
674	* - Perhaps: CPUMCTX.fXStateMask (windows only)
675	*
676	* We currently ASSUME that neither CPUMCTX_WSF_IBPB_ENTRY nor
677	* CPUMCTX_WSF_IBPB_EXIT cannot be changed at runtime.
678	*/
679	static void hmR0SvmUpdateVmRunFunction(PVMCPUCC pVCpu)
680	{
681	static const struct CLANGWORKAROUND { PFNHMSVMVMRUN pfn; } s_aHmR0SvmVmRunFunctions[] =
682	{
683	{ hmR0SvmVmRun_SansXcr0_SansIbpbEntry_SansIbpbExit },
684	{ hmR0SvmVmRun_WithXcr0_SansIbpbEntry_SansIbpbExit },
685	{ hmR0SvmVmRun_SansXcr0_WithIbpbEntry_SansIbpbExit },
686	{ hmR0SvmVmRun_WithXcr0_WithIbpbEntry_SansIbpbExit },
687	{ hmR0SvmVmRun_SansXcr0_SansIbpbEntry_WithIbpbExit },
688	{ hmR0SvmVmRun_WithXcr0_SansIbpbEntry_WithIbpbExit },
689	{ hmR0SvmVmRun_SansXcr0_WithIbpbEntry_WithIbpbExit },
690	{ hmR0SvmVmRun_WithXcr0_WithIbpbEntry_WithIbpbExit },
691	};
692	uintptr_t const idx = (pVCpu->hmr0.s.fLoadSaveGuestXcr0 ? 1 : 0)
693	\| (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_IBPB_ENTRY ? 2 : 0)
694	\| (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_IBPB_EXIT ? 4 : 0);
695	PFNHMSVMVMRUN const pfnVMRun = s_aHmR0SvmVmRunFunctions[idx].pfn;
696	if (pVCpu->hmr0.s.svm.pfnVMRun != pfnVMRun)
697	pVCpu->hmr0.s.svm.pfnVMRun = pfnVMRun;
698	}
699
700
701	/**
702	* Selector FNHMSVMVMRUN implementation.
703	*/
704	static DECLCALLBACK(int) hmR0SvmVMRunSelector(PVMCC pVM, PVMCPUCC pVCpu, RTHCPHYS HCPhysVMCB)
705	{
706	hmR0SvmUpdateVmRunFunction(pVCpu);
707	return pVCpu->hmr0.s.svm.pfnVMRun(pVM, pVCpu, HCPhysVMCB);
708	}
709
710
711	/**
712	* Does per-VM AMD-V initialization.
713	*
714	* @returns VBox status code.
715	* @param pVM The cross context VM structure.
716	*/
717	VMMR0DECL(int) SVMR0InitVM(PVMCC pVM)
718	{
719	int rc = VERR_INTERNAL_ERROR_5;
720
721	/*
722	* Check for an AMD CPU erratum which requires us to flush the TLB before every world-switch.
723	*/
724	uint32_t u32Family;
725	uint32_t u32Model;
726	uint32_t u32Stepping;
727	if (HMIsSubjectToSvmErratum170(&u32Family, &u32Model, &u32Stepping))
728	{
729	Log4Func(("AMD cpu with erratum 170 family %#x model %#x stepping %#x\n", u32Family, u32Model, u32Stepping));
730	pVM->hmr0.s.svm.fAlwaysFlushTLB = true;
731	}
732
733	/*
734	* Initialize the R0 memory objects up-front so we can properly cleanup on allocation failures.
735	*/
736	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
737	{
738	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
739	pVCpu->hmr0.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
740	pVCpu->hmr0.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
741	pVCpu->hmr0.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
742	}
743
744	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
745	{
746	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
747
748	/*
749	* Initialize the hardware-assisted SVM guest-execution handler.
750	* We now use a single handler for both 32-bit and 64-bit guests, see @bugref{6208#c73}.
751	*/
752	pVCpu->hmr0.s.svm.pfnVMRun = hmR0SvmVMRunSelector;
753
754	/*
755	* Allocate one page for the host-context VM control block (VMCB). This is used for additional host-state (such as
756	* FS, GS, Kernel GS Base, etc.) apart from the host-state save area specified in MSR_K8_VM_HSAVE_PA.
757	*/
758	/** @todo Does this need to be below 4G? */
759	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjVmcbHost, SVM_VMCB_PAGES << HOST_PAGE_SHIFT, false /* fExecutable */);
760	if (RT_FAILURE(rc))
761	goto failure_cleanup;
762
763	void *pvVmcbHost = RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjVmcbHost);
764	pVCpu->hmr0.s.svm.HCPhysVmcbHost = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjVmcbHost, 0 /* iPage */);
765	Assert(pVCpu->hmr0.s.svm.HCPhysVmcbHost < _4G);
766	RT_BZERO(pvVmcbHost, HOST_PAGE_SIZE);
767
768	/*
769	* Allocate one page for the guest-state VMCB.
770	*/
771	/** @todo Does this need to be below 4G? */
772	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjVmcb, SVM_VMCB_PAGES << HOST_PAGE_SHIFT, false /* fExecutable */);
773	if (RT_FAILURE(rc))
774	goto failure_cleanup;
775
776	pVCpu->hmr0.s.svm.pVmcb = (PSVMVMCB)RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjVmcb);
777	pVCpu->hmr0.s.svm.HCPhysVmcb = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjVmcb, 0 /* iPage */);
778	Assert(pVCpu->hmr0.s.svm.HCPhysVmcb < _4G);
779	RT_BZERO(pVCpu->hmr0.s.svm.pVmcb, HOST_PAGE_SIZE);
780
781	/*
782	* Allocate two pages (8 KB) for the MSR permission bitmap. There doesn't seem to be a way to convince
783	* SVM to not require one.
784	*/
785	/** @todo Does this need to be below 4G? */
786	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjMsrBitmap, SVM_MSRPM_PAGES << HOST_PAGE_SHIFT,
787	false /* fExecutable */);
788	if (RT_FAILURE(rc))
789	goto failure_cleanup;
790
791	pVCpu->hmr0.s.svm.pvMsrBitmap = RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjMsrBitmap);
792	pVCpu->hmr0.s.svm.HCPhysMsrBitmap = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjMsrBitmap, 0 /* iPage */);
793	/* Set all bits to intercept all MSR accesses (changed later on). */
794	ASMMemFill32(pVCpu->hmr0.s.svm.pvMsrBitmap, SVM_MSRPM_PAGES << HOST_PAGE_SHIFT, UINT32_C(0xffffffff));
795	}
796
797	return VINF_SUCCESS;
798
799	failure_cleanup:
800	hmR0SvmFreeStructs(pVM);
801	return rc;
802	}
803
804
805	/**
806	* Does per-VM AMD-V termination.
807	*
808	* @returns VBox status code.
809	* @param pVM The cross context VM structure.
810	*/
811	VMMR0DECL(int) SVMR0TermVM(PVMCC pVM)
812	{
813	hmR0SvmFreeStructs(pVM);
814	return VINF_SUCCESS;
815	}
816
817
818	/**
819	* Returns whether the VMCB Clean Bits feature is supported.
820	*
821	* @returns @c true if supported, @c false otherwise.
822	* @param pVCpu The cross context virtual CPU structure.
823	* @param fIsNestedGuest Whether we are currently executing the nested-guest.
824	*/
825	DECL_FORCE_INLINE(bool) hmR0SvmSupportsVmcbCleanBits(PVMCPUCC pVCpu, bool fIsNestedGuest)
826	{
827	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
828	bool const fHostVmcbCleanBits = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VMCB_CLEAN);
829	if (!fIsNestedGuest)
830	return fHostVmcbCleanBits;
831	return fHostVmcbCleanBits && pVM->cpum.ro.GuestFeatures.fSvmVmcbClean;
832	}
833
834
835	/**
836	* Returns whether the decode assists feature is supported.
837	*
838	* @returns @c true if supported, @c false otherwise.
839	* @param pVCpu The cross context virtual CPU structure.
840	*/
841	DECLINLINE(bool) hmR0SvmSupportsDecodeAssists(PVMCPUCC pVCpu)
842	{
843	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
844	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
845	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
846	return (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS)
847	&& pVM->cpum.ro.GuestFeatures.fSvmDecodeAssists;
848	#endif
849	return RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS);
850	}
851
852
853	/**
854	* Returns whether the NRIP_SAVE feature is supported.
855	*
856	* @returns @c true if supported, @c false otherwise.
857	* @param pVCpu The cross context virtual CPU structure.
858	*/
859	DECLINLINE(bool) hmR0SvmSupportsNextRipSave(PVMCPUCC pVCpu)
860	{
861	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
862	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
863	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
864	return (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE)
865	&& pVM->cpum.ro.GuestFeatures.fSvmNextRipSave;
866	#endif
867	return RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE);
868	}
869
870
871	/**
872	* Sets the permission bits for the specified MSR in the MSRPM bitmap.
873	*
874	* @param pVCpu The cross context virtual CPU structure.
875	* @param pbMsrBitmap Pointer to the MSR bitmap.
876	* @param idMsr The MSR for which the permissions are being set.
877	* @param enmRead MSR read permissions.
878	* @param enmWrite MSR write permissions.
879	*
880	* @remarks This function does -not- clear the VMCB clean bits for MSRPM. The
881	* caller needs to take care of this.
882	*/
883	static void hmR0SvmSetMsrPermission(PVMCPUCC pVCpu, uint8_t *pbMsrBitmap, uint32_t idMsr, SVMMSREXITREAD enmRead,
884	SVMMSREXITWRITE enmWrite)
885	{
886	bool const fInNestedGuestMode = CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx);
887	uint16_t offMsrpm;
888	uint8_t uMsrpmBit;
889	int rc = CPUMGetSvmMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
890	AssertRC(rc);
891
892	Assert(uMsrpmBit == 0 \|\| uMsrpmBit == 2 \|\| uMsrpmBit == 4 \|\| uMsrpmBit == 6);
893	Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
894
895	pbMsrBitmap += offMsrpm;
896	if (enmRead == SVMMSREXIT_INTERCEPT_READ)
897	*pbMsrBitmap \|= RT_BIT(uMsrpmBit);
898	else
899	{
900	if (!fInNestedGuestMode)
901	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
902	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
903	else
904	{
905	/* Only clear the bit if the nested-guest is also not intercepting the MSR read.*/
906	if (!(pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm] & RT_BIT(uMsrpmBit)))
907	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
908	else
909	Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit));
910	}
911	#endif
912	}
913
914	if (enmWrite == SVMMSREXIT_INTERCEPT_WRITE)
915	*pbMsrBitmap \|= RT_BIT(uMsrpmBit + 1);
916	else
917	{
918	if (!fInNestedGuestMode)
919	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
920	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
921	else
922	{
923	/* Only clear the bit if the nested-guest is also not intercepting the MSR write.*/
924	if (!(pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm] & RT_BIT(uMsrpmBit + 1)))
925	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
926	else
927	Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
928	}
929	#endif
930	}
931	}
932
933
934	/**
935	* Sets up AMD-V for the specified VM.
936	* This function is only called once per-VM during initalization.
937	*
938	* @returns VBox status code.
939	* @param pVM The cross context VM structure.
940	*/
941	VMMR0DECL(int) SVMR0SetupVM(PVMCC pVM)
942	{
943	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
944	AssertReturn(pVM, VERR_INVALID_PARAMETER);
945
946	/*
947	* Validate and copy over some parameters.
948	*/
949	AssertReturn(pVM->hm.s.svm.fSupported, VERR_INCOMPATIBLE_CONFIG);
950	bool const fNestedPaging = pVM->hm.s.fNestedPagingCfg;
951	AssertReturn(!fNestedPaging \|\| (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NESTED_PAGING), VERR_INCOMPATIBLE_CONFIG);
952	pVM->hmr0.s.fNestedPaging = fNestedPaging;
953	pVM->hmr0.s.fAllow64BitGuests = pVM->hm.s.fAllow64BitGuestsCfg;
954
955	/*
956	* Determin some configuration parameters.
957	*/
958	bool const fPauseFilter = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER);
959	bool const fPauseFilterThreshold = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER_THRESHOLD);
960	bool const fUsePauseFilter = fPauseFilter && pVM->hm.s.svm.cPauseFilter;
961
962	bool const fLbrVirt = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_LBR_VIRT);
963	bool const fUseLbrVirt = fLbrVirt && pVM->hm.s.svm.fLbrVirt; /** @todo IEM implementation etc. */
964
965	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
966	bool const fVirtVmsaveVmload = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VIRT_VMSAVE_VMLOAD);
967	bool const fUseVirtVmsaveVmload = fVirtVmsaveVmload && pVM->hm.s.svm.fVirtVmsaveVmload && fNestedPaging;
968
969	bool const fVGif = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VGIF);
970	bool const fUseVGif = fVGif && pVM->hm.s.svm.fVGif;
971	#endif
972
973	PVMCPUCC pVCpu0 = VMCC_GET_CPU_0(pVM);
974	PSVMVMCB pVmcb0 = pVCpu0->hmr0.s.svm.pVmcb;
975	AssertMsgReturn(RT_VALID_PTR(pVmcb0), ("Invalid pVmcb (%p) for vcpu[0]\n", pVmcb0), VERR_SVM_INVALID_PVMCB);
976	PSVMVMCBCTRL pVmcbCtrl0 = &pVmcb0->ctrl;
977
978	/* Always trap #AC for reasons of security. */
979	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_AC);
980
981	/* Always trap #DB for reasons of security. */
982	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_DB);
983
984	/* Trap exceptions unconditionally (debug purposes). */
985	#ifdef HMSVM_ALWAYS_TRAP_PF
986	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_PF);
987	#endif
988	#ifdef HMSVM_ALWAYS_TRAP_ALL_XCPTS
989	/* If you add any exceptions here, make sure to update hmR0SvmHandleExit(). */
990	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_BP)
991	\| RT_BIT_32(X86_XCPT_DE)
992	\| RT_BIT_32(X86_XCPT_NM)
993	\| RT_BIT_32(X86_XCPT_UD)
994	\| RT_BIT_32(X86_XCPT_NP)
995	\| RT_BIT_32(X86_XCPT_SS)
996	\| RT_BIT_32(X86_XCPT_GP)
997	\| RT_BIT_32(X86_XCPT_PF)
998	\| RT_BIT_32(X86_XCPT_MF)
999	;
1000	#endif
1001
1002	/* Apply the exceptions intercepts needed by the GIM provider. */
1003	if (pVCpu0->hm.s.fGIMTrapXcptUD \|\| pVCpu0->hm.s.svm.fEmulateLongModeSysEnterExit)
1004	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_UD);
1005
1006	/* Apply the exceptions intercepts needed by the GCM fixers. */
1007	if (pVCpu0->hm.s.fGCMTrapXcptDE)
1008	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_DE);
1009
1010	/* The mesa 3d driver hack needs #GP. */
1011	if (pVCpu0->hm.s.fTrapXcptGpForLovelyMesaDrv)
1012	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_GP);
1013
1014	/* Set up unconditional intercepts and conditions. */
1015	pVmcbCtrl0->u64InterceptCtrl = HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS
1016	\| SVM_CTRL_INTERCEPT_VMMCALL
1017	\| SVM_CTRL_INTERCEPT_VMSAVE
1018	\| SVM_CTRL_INTERCEPT_VMLOAD
1019	\| SVM_CTRL_INTERCEPT_CLGI
1020	\| SVM_CTRL_INTERCEPT_STGI;
1021
1022	#ifdef HMSVM_ALWAYS_TRAP_TASK_SWITCH
1023	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_TASK_SWITCH;
1024	#endif
1025
1026	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1027	if (pVCpu0->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvm)
1028	{
1029	/* Virtualized VMSAVE/VMLOAD. */
1030	if (fUseVirtVmsaveVmload)
1031	{
1032	pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload = 1;
1033	pVmcbCtrl0->u64InterceptCtrl &= ~( SVM_CTRL_INTERCEPT_VMSAVE
1034	\| SVM_CTRL_INTERCEPT_VMLOAD);
1035	}
1036	else
1037	Assert(!pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload);
1038
1039	/* Virtual GIF. */
1040	if (fUseVGif)
1041	{
1042	pVmcbCtrl0->IntCtrl.n.u1VGifEnable = 1;
1043	pVmcbCtrl0->u64InterceptCtrl &= ~( SVM_CTRL_INTERCEPT_CLGI
1044	\| SVM_CTRL_INTERCEPT_STGI);
1045	}
1046	else
1047	Assert(!pVmcbCtrl0->IntCtrl.n.u1VGifEnable);
1048	}
1049	else
1050	#endif
1051	{
1052	Assert(!pVCpu0->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvm);
1053	Assert(!pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload);
1054	Assert(!pVmcbCtrl0->IntCtrl.n.u1VGifEnable);
1055	}
1056
1057	/* CR4 writes must always be intercepted for tracking PGM mode changes. */
1058	pVmcbCtrl0->u16InterceptWrCRx = RT_BIT(4);
1059
1060	/* Intercept all DRx reads and writes by default. Changed later on. */
1061	pVmcbCtrl0->u16InterceptRdDRx = 0xffff;
1062	pVmcbCtrl0->u16InterceptWrDRx = 0xffff;
1063
1064	/* Virtualize masking of INTR interrupts. (reads/writes from/to CR8 go to the V_TPR register) */
1065	pVmcbCtrl0->IntCtrl.n.u1VIntrMasking = 1;
1066
1067	/* Ignore the priority in the virtual TPR. This is necessary for delivering PIC style (ExtInt) interrupts
1068	and we currently deliver both PIC and APIC interrupts alike, see hmR0SvmEvaluatePendingEvent() */
1069	pVmcbCtrl0->IntCtrl.n.u1IgnoreTPR = 1;
1070
1071	/* Set the IO permission bitmap physical addresses. */
1072	pVmcbCtrl0->u64IOPMPhysAddr = g_HCPhysIOBitmap;
1073
1074	/* LBR virtualization. */
1075	pVmcbCtrl0->LbrVirt.n.u1LbrVirt = fUseLbrVirt;
1076
1077	/* The host ASID MBZ, for the guest start with 1. */
1078	pVmcbCtrl0->TLBCtrl.n.u32ASID = 1;
1079
1080	/* Setup Nested Paging. This doesn't change throughout the execution time of the VM. */
1081	pVmcbCtrl0->NestedPagingCtrl.n.u1NestedPaging = fNestedPaging;
1082
1083	/* Without Nested Paging, we need additionally intercepts. */
1084	if (!fNestedPaging)
1085	{
1086	/* CR3 reads/writes must be intercepted; our shadow values differ from the guest values. */
1087	pVmcbCtrl0->u16InterceptRdCRx \|= RT_BIT(3);
1088	pVmcbCtrl0->u16InterceptWrCRx \|= RT_BIT(3);
1089
1090	/* Intercept INVLPG and task switches (may change CR3, EFLAGS, LDT). */
1091	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_INVLPG
1092	\| SVM_CTRL_INTERCEPT_TASK_SWITCH;
1093
1094	/* Page faults must be intercepted to implement shadow paging. */
1095	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_PF);
1096	}
1097
1098	/* Workaround for missing OS/2 TLB flush, see ticketref:20625. */
1099	if (pVM->hm.s.fMissingOS2TlbFlushWorkaround)
1100	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_TR_WRITES;
1101
1102	/* Setup Pause Filter for guest pause-loop (spinlock) exiting. */
1103	if (fUsePauseFilter)
1104	{
1105	Assert(pVM->hm.s.svm.cPauseFilter > 0);
1106	pVmcbCtrl0->u16PauseFilterCount = pVM->hm.s.svm.cPauseFilter;
1107	if (fPauseFilterThreshold)
1108	pVmcbCtrl0->u16PauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
1109	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_PAUSE;
1110	}
1111
1112	/*
1113	* Setup the MSR permission bitmap.
1114	* The following MSRs are saved/restored automatically during the world-switch.
1115	* Don't intercept guest read/write accesses to these MSRs.
1116	*/
1117	uint8_t pbMsrBitmap0 = (uint8_t )pVCpu0->hmr0.s.svm.pvMsrBitmap;
1118	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1119	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_CSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1120	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K6_STAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1121	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_SF_MASK, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1122	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_FS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1123	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1124	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_KERNEL_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1125	if (!pVCpu0->hm.s.svm.fEmulateLongModeSysEnterExit)
1126	{
1127	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_CS, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1128	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1129	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1130	}
1131	else
1132	{
1133	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_CS, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1134	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1135	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1136	}
1137	pVmcbCtrl0->u64MSRPMPhysAddr = pVCpu0->hmr0.s.svm.HCPhysMsrBitmap;
1138
1139	/* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
1140	Assert(pVmcbCtrl0->u32VmcbCleanBits == 0);
1141
1142	for (VMCPUID idCpu = 1; idCpu < pVM->cCpus; idCpu++)
1143	{
1144	PVMCPUCC pVCpuCur = VMCC_GET_CPU(pVM, idCpu);
1145	PSVMVMCB pVmcbCur = pVCpuCur->hmr0.s.svm.pVmcb;
1146	AssertMsgReturn(RT_VALID_PTR(pVmcbCur), ("Invalid pVmcb (%p) for vcpu[%u]\n", pVmcbCur, idCpu), VERR_SVM_INVALID_PVMCB);
1147	PSVMVMCBCTRL pVmcbCtrlCur = &pVmcbCur->ctrl;
1148
1149	/* Copy the VMCB control area. */
1150	memcpy(pVmcbCtrlCur, pVmcbCtrl0, sizeof(*pVmcbCtrlCur));
1151
1152	/* Copy the MSR bitmap and setup the VCPU-specific host physical address. */
1153	uint8_t pbMsrBitmapCur = (uint8_t )pVCpuCur->hmr0.s.svm.pvMsrBitmap;
1154	memcpy(pbMsrBitmapCur, pbMsrBitmap0, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
1155	pVmcbCtrlCur->u64MSRPMPhysAddr = pVCpuCur->hmr0.s.svm.HCPhysMsrBitmap;
1156
1157	/* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
1158	Assert(pVmcbCtrlCur->u32VmcbCleanBits == 0);
1159
1160	/* Verify our assumption that GIM providers trap #UD uniformly across VCPUs initially. */
1161	Assert(pVCpuCur->hm.s.fGIMTrapXcptUD == pVCpu0->hm.s.fGIMTrapXcptUD);
1162	/* Same for GCM, #DE trapping should be uniform across VCPUs. */
1163	Assert(pVCpuCur->hm.s.fGCMTrapXcptDE == pVCpu0->hm.s.fGCMTrapXcptDE);
1164	}
1165
1166	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1167	LogRel(("HM: fUsePauseFilter=%RTbool fUseLbrVirt=%RTbool fUseVGif=%RTbool fUseVirtVmsaveVmload=%RTbool\n", fUsePauseFilter,
1168	fUseLbrVirt, fUseVGif, fUseVirtVmsaveVmload));
1169	#else
1170	LogRel(("HM: fUsePauseFilter=%RTbool fUseLbrVirt=%RTbool\n", fUsePauseFilter, fUseLbrVirt));
1171	#endif
1172	return VINF_SUCCESS;
1173	}
1174
1175
1176	/**
1177	* Gets a pointer to the currently active guest (or nested-guest) VMCB.
1178	*
1179	* @returns Pointer to the current context VMCB.
1180	* @param pVCpu The cross context virtual CPU structure.
1181	*/
1182	DECLINLINE(PSVMVMCB) hmR0SvmGetCurrentVmcb(PVMCPUCC pVCpu)
1183	{
1184	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1185	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
1186	return &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
1187	#endif
1188	return pVCpu->hmr0.s.svm.pVmcb;
1189	}
1190
1191
1192	/**
1193	* Gets a pointer to the nested-guest VMCB cache.
1194	*
1195	* @returns Pointer to the nested-guest VMCB cache.
1196	* @param pVCpu The cross context virtual CPU structure.
1197	*/
1198	DECLINLINE(PSVMNESTEDVMCBCACHE) hmR0SvmGetNestedVmcbCache(PVMCPUCC pVCpu)
1199	{
1200	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1201	Assert(pVCpu->hm.s.svm.NstGstVmcbCache.fCacheValid);
1202	return &pVCpu->hm.s.svm.NstGstVmcbCache;
1203	#else
1204	RT_NOREF(pVCpu);
1205	return NULL;
1206	#endif
1207	}
1208
1209
1210	/**
1211	* Invalidates a guest page by guest virtual address.
1212	*
1213	* @returns VBox status code.
1214	* @param pVCpu The cross context virtual CPU structure.
1215	* @param GCVirt Guest virtual address of the page to invalidate.
1216	*/
1217	VMMR0DECL(int) SVMR0InvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCVirt)
1218	{
1219	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fSupported);
1220
1221	bool const fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH) \|\| pVCpu->CTX_SUFF(pVM)->hmr0.s.svm.fAlwaysFlushTLB;
1222
1223	/* Skip it if a TLB flush is already pending. */
1224	if (!fFlushPending)
1225	{
1226	Log4Func(("%#RGv\n", GCVirt));
1227
1228	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
1229	AssertMsgReturn(pVmcb, ("Invalid pVmcb!\n"), VERR_SVM_INVALID_PVMCB);
1230
1231	SVMR0InvlpgA(GCVirt, pVmcb->ctrl.TLBCtrl.n.u32ASID);
1232	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1233	}
1234	return VINF_SUCCESS;
1235	}
1236
1237
1238	/**
1239	* Flushes the appropriate tagged-TLB entries.
1240	*
1241	* @param pHostCpu The HM physical-CPU structure.
1242	* @param pVCpu The cross context virtual CPU structure.
1243	* @param pVmcb Pointer to the VM control block.
1244	*/
1245	static void hmR0SvmFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1246	{
1247	/*
1248	* Force a TLB flush for the first world switch if the current CPU differs from the one
1249	* we ran on last. This can happen both for start & resume due to long jumps back to
1250	* ring-3.
1251	*
1252	* We also force a TLB flush every time when executing a nested-guest VCPU as there is no
1253	* correlation between it and the physical CPU.
1254	*
1255	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while
1256	* flushing the TLB, so we cannot reuse the ASIDs without flushing.
1257	*/
1258	bool fNewAsid = false;
1259	Assert(pHostCpu->idCpu != NIL_RTCPUID);
1260	if ( pVCpu->hmr0.s.idLastCpu != pHostCpu->idCpu
1261	\|\| pVCpu->hmr0.s.cTlbFlushes != pHostCpu->cTlbFlushes
1262	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1263	\|\| CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx)
1264	#endif
1265	)
1266	{
1267	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1268	pVCpu->hmr0.s.fForceTLBFlush = true;
1269	fNewAsid = true;
1270	}
1271
1272	/* Set TLB flush state as checked until we return from the world switch. */
1273	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true);
1274
1275	/* Check for explicit TLB flushes. */
1276	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1277	{
1278	pVCpu->hmr0.s.fForceTLBFlush = true;
1279	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1280	}
1281
1282	/*
1283	* If the AMD CPU erratum 170, We need to flush the entire TLB for each world switch. Sad.
1284	* This Host CPU requirement takes precedence.
1285	*/
1286	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1287	if (pVM->hmr0.s.svm.fAlwaysFlushTLB)
1288	{
1289	pHostCpu->uCurrentAsid = 1;
1290	pVCpu->hmr0.s.uCurrentAsid = 1;
1291	pVCpu->hmr0.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1292	pVCpu->hmr0.s.idLastCpu = pHostCpu->idCpu;
1293	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1294
1295	/* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1296	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1297	}
1298	else
1299	{
1300	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_NOTHING;
1301	if (pVCpu->hmr0.s.fForceTLBFlush)
1302	{
1303	/* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1304	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1305
1306	if (fNewAsid)
1307	{
1308	++pHostCpu->uCurrentAsid;
1309
1310	bool fHitASIDLimit = false;
1311	if (pHostCpu->uCurrentAsid >= g_uHmMaxAsid)
1312	{
1313	pHostCpu->uCurrentAsid = 1; /* Wraparound at 1; host uses 0 */
1314	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new ASID. */
1315	fHitASIDLimit = true;
1316	}
1317
1318	if ( fHitASIDLimit
1319	\|\| pHostCpu->fFlushAsidBeforeUse)
1320	{
1321	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1322	pHostCpu->fFlushAsidBeforeUse = false;
1323	}
1324
1325	pVCpu->hmr0.s.uCurrentAsid = pHostCpu->uCurrentAsid;
1326	pVCpu->hmr0.s.idLastCpu = pHostCpu->idCpu;
1327	pVCpu->hmr0.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1328	}
1329	else
1330	{
1331	if (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_FLUSH_BY_ASID)
1332	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_SINGLE_CONTEXT;
1333	else
1334	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1335	}
1336
1337	pVCpu->hmr0.s.fForceTLBFlush = false;
1338	}
1339	}
1340
1341	/* Update VMCB with the ASID. */
1342	if (pVmcb->ctrl.TLBCtrl.n.u32ASID != pVCpu->hmr0.s.uCurrentAsid)
1343	{
1344	pVmcb->ctrl.TLBCtrl.n.u32ASID = pVCpu->hmr0.s.uCurrentAsid;
1345	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_ASID;
1346	}
1347
1348	AssertMsg(pVCpu->hmr0.s.idLastCpu == pHostCpu->idCpu,
1349	("vcpu idLastCpu=%u hostcpu idCpu=%u\n", pVCpu->hmr0.s.idLastCpu, pHostCpu->idCpu));
1350	AssertMsg(pVCpu->hmr0.s.cTlbFlushes == pHostCpu->cTlbFlushes,
1351	("Flush count mismatch for cpu %u (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hmr0.s.cTlbFlushes, pHostCpu->cTlbFlushes));
1352	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < g_uHmMaxAsid,
1353	("cpu%d uCurrentAsid = %x\n", pHostCpu->idCpu, pHostCpu->uCurrentAsid));
1354	AssertMsg(pVCpu->hmr0.s.uCurrentAsid >= 1 && pVCpu->hmr0.s.uCurrentAsid < g_uHmMaxAsid,
1355	("cpu%d VM uCurrentAsid = %x\n", pHostCpu->idCpu, pVCpu->hmr0.s.uCurrentAsid));
1356
1357	#ifdef VBOX_WITH_STATISTICS
1358	if (pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_NOTHING)
1359	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1360	else if ( pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT
1361	\|\| pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT_RETAIN_GLOBALS)
1362	{
1363	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1364	}
1365	else
1366	{
1367	Assert(pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_ENTIRE);
1368	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushEntire);
1369	}
1370	#endif
1371	}
1372
1373
1374	/**
1375	* Sets an exception intercept in the specified VMCB.
1376	*
1377	* @param pVmcb Pointer to the VM control block.
1378	* @param uXcpt The exception (X86_XCPT_*).
1379	*/
1380	DECLINLINE(void) hmR0SvmSetXcptIntercept(PSVMVMCB pVmcb, uint8_t uXcpt)
1381	{
1382	if (!(pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt)))
1383	{
1384	pVmcb->ctrl.u32InterceptXcpt \|= RT_BIT(uXcpt);
1385	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1386	}
1387	}
1388
1389
1390	/**
1391	* Clears an exception intercept in the specified VMCB.
1392	*
1393	* @param pVCpu The cross context virtual CPU structure.
1394	* @param pVmcb Pointer to the VM control block.
1395	* @param uXcpt The exception (X86_XCPT_*).
1396	*
1397	* @remarks This takes into account if we're executing a nested-guest and only
1398	* removes the exception intercept if both the guest -and- nested-guest
1399	* are not intercepting it.
1400	*/
1401	DECLINLINE(void) hmR0SvmClearXcptIntercept(PVMCPUCC pVCpu, PSVMVMCB pVmcb, uint8_t uXcpt)
1402	{
1403	Assert(uXcpt != X86_XCPT_DB);
1404	Assert(uXcpt != X86_XCPT_AC);
1405	Assert(uXcpt != X86_XCPT_GP);
1406	#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
1407	if (pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt))
1408	{
1409	bool fRemove = true;
1410	# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1411	/* Only remove the intercept if the nested-guest is also not intercepting it! */
1412	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1413	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1414	{
1415	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1416	fRemove = !(pVmcbNstGstCache->u32InterceptXcpt & RT_BIT(uXcpt));
1417	}
1418	# else
1419	RT_NOREF(pVCpu);
1420	# endif
1421	if (fRemove)
1422	{
1423	pVmcb->ctrl.u32InterceptXcpt &= ~RT_BIT(uXcpt);
1424	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1425	}
1426	}
1427	#else
1428	RT_NOREF3(pVCpu, pVmcb, uXcpt);
1429	#endif
1430	}
1431
1432
1433	/**
1434	* Sets a control intercept in the specified VMCB.
1435	*
1436	* @param pVmcb Pointer to the VM control block.
1437	* @param fCtrlIntercept The control intercept (SVM_CTRL_INTERCEPT_*).
1438	*/
1439	DECLINLINE(void) hmR0SvmSetCtrlIntercept(PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
1440	{
1441	if (!(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept))
1442	{
1443	pVmcb->ctrl.u64InterceptCtrl \|= fCtrlIntercept;
1444	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1445	}
1446	}
1447
1448
1449	/**
1450	* Clears a control intercept in the specified VMCB.
1451	*
1452	* @returns @c true if the intercept is still set, @c false otherwise.
1453	* @param pVCpu The cross context virtual CPU structure.
1454	* @param pVmcb Pointer to the VM control block.
1455	* @param fCtrlIntercept The control intercept (SVM_CTRL_INTERCEPT_*).
1456	*
1457	* @remarks This takes into account if we're executing a nested-guest and only
1458	* removes the control intercept if both the guest -and- nested-guest
1459	* are not intercepting it.
1460	*/
1461	static bool hmR0SvmClearCtrlIntercept(PVMCPUCC pVCpu, PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
1462	{
1463	if (pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept)
1464	{
1465	bool fRemove = true;
1466	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1467	/* Only remove the control intercept if the nested-guest is also not intercepting it! */
1468	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
1469	{
1470	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1471	fRemove = !(pVmcbNstGstCache->u64InterceptCtrl & fCtrlIntercept);
1472	}
1473	#else
1474	RT_NOREF(pVCpu);
1475	#endif
1476	if (fRemove)
1477	{
1478	pVmcb->ctrl.u64InterceptCtrl &= ~fCtrlIntercept;
1479	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1480	}
1481	}
1482
1483	return RT_BOOL(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept);
1484	}
1485
1486
1487	/**
1488	* Exports the guest (or nested-guest) CR0 into the VMCB.
1489	*
1490	* @param pVCpu The cross context virtual CPU structure.
1491	* @param pVmcb Pointer to the VM control block.
1492	*
1493	* @remarks This assumes we always pre-load the guest FPU.
1494	* @remarks No-long-jump zone!!!
1495	*/
1496	static void hmR0SvmExportGuestCR0(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1497	{
1498	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1499
1500	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1501	uint64_t const uGuestCr0 = pCtx->cr0;
1502	uint64_t uShadowCr0 = uGuestCr0;
1503
1504	/* Always enable caching. */
1505	uShadowCr0 &= ~(X86_CR0_CD \| X86_CR0_NW);
1506
1507	/* When Nested Paging is not available use shadow page tables and intercept #PFs (latter done in SVMR0SetupVM()). */
1508	if (!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
1509	{
1510	uShadowCr0 \|= X86_CR0_PG /* Use shadow page tables. */
1511	\| X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF #VMEXIT. */
1512	}
1513
1514	/*
1515	* Use the #MF style of legacy-FPU error reporting for now. Although AMD-V has MSRs that
1516	* lets us isolate the host from it, IEM/REM still needs work to emulate it properly,
1517	* see @bugref{7243#c103}.
1518	*/
1519	if (!(uGuestCr0 & X86_CR0_NE))
1520	{
1521	uShadowCr0 \|= X86_CR0_NE;
1522	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_MF);
1523	}
1524	else
1525	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_MF);
1526
1527	/*
1528	* If the shadow and guest CR0 are identical we can avoid intercepting CR0 reads.
1529	*
1530	* CR0 writes still needs interception as PGM requires tracking paging mode changes,
1531	* see @bugref{6944}.
1532	*
1533	* We also don't ever want to honor weird things like cache disable from the guest.
1534	* However, we can avoid intercepting changes to the TS & MP bits by clearing the CR0
1535	* write intercept below and keeping SVM_CTRL_INTERCEPT_CR0_SEL_WRITE instead.
1536	*/
1537	if (uShadowCr0 == uGuestCr0)
1538	{
1539	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1540	{
1541	pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(0);
1542	pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(0);
1543	Assert(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_CR0_SEL_WRITE);
1544	}
1545	else
1546	{
1547	/* If the nested-hypervisor intercepts CR0 reads/writes, we need to continue intercepting them. */
1548	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1549	pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx & ~RT_BIT(0))
1550	\| (pVmcbNstGstCache->u16InterceptRdCRx & RT_BIT(0));
1551	pVmcb->ctrl.u16InterceptWrCRx = (pVmcb->ctrl.u16InterceptWrCRx & ~RT_BIT(0))
1552	\| (pVmcbNstGstCache->u16InterceptWrCRx & RT_BIT(0));
1553	}
1554	}
1555	else
1556	{
1557	pVmcb->ctrl.u16InterceptRdCRx \|= RT_BIT(0);
1558	pVmcb->ctrl.u16InterceptWrCRx \|= RT_BIT(0);
1559	}
1560	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1561
1562	Assert(!RT_HI_U32(uShadowCr0));
1563	if (pVmcb->guest.u64CR0 != uShadowCr0)
1564	{
1565	pVmcb->guest.u64CR0 = uShadowCr0;
1566	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1567	}
1568	}
1569
1570
1571	/**
1572	* Exports the guest (or nested-guest) CR3 into the VMCB.
1573	*
1574	* @param pVCpu The cross context virtual CPU structure.
1575	* @param pVmcb Pointer to the VM control block.
1576	*
1577	* @remarks No-long-jump zone!!!
1578	*/
1579	static void hmR0SvmExportGuestCR3(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1580	{
1581	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1582
1583	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1584	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1585	if (pVM->hmr0.s.fNestedPaging)
1586	{
1587	pVmcb->ctrl.u64NestedPagingCR3 = PGMGetHyperCR3(pVCpu);
1588	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1589	pVmcb->guest.u64CR3 = pCtx->cr3;
1590	Assert(pVmcb->ctrl.u64NestedPagingCR3);
1591	}
1592	else
1593	pVmcb->guest.u64CR3 = PGMGetHyperCR3(pVCpu);
1594
1595	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1596	}
1597
1598
1599	/**
1600	* Exports the guest (or nested-guest) CR4 into the VMCB.
1601	*
1602	* @param pVCpu The cross context virtual CPU structure.
1603	* @param pVmcb Pointer to the VM control block.
1604	*
1605	* @remarks No-long-jump zone!!!
1606	*/
1607	static int hmR0SvmExportGuestCR4(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1608	{
1609	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1610
1611	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1612	uint64_t uShadowCr4 = pCtx->cr4;
1613	if (!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
1614	{
1615	switch (pVCpu->hm.s.enmShadowMode)
1616	{
1617	case PGMMODE_REAL:
1618	case PGMMODE_PROTECTED: /* Protected mode, no paging. */
1619	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1620
1621	case PGMMODE_32_BIT: /* 32-bit paging. */
1622	uShadowCr4 &= ~X86_CR4_PAE;
1623	break;
1624
1625	case PGMMODE_PAE: /* PAE paging. */
1626	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1627	/** Must use PAE paging as we could use physical memory > 4 GB */
1628	uShadowCr4 \|= X86_CR4_PAE;
1629	break;
1630
1631	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1632	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1633	#ifdef VBOX_WITH_64_BITS_GUESTS
1634	break;
1635	#else
1636	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1637	#endif
1638
1639	default: /* shut up gcc */
1640	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1641	}
1642	}
1643
1644	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
1645	bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
1646	if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
1647	{
1648	pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
1649	hmR0SvmUpdateVmRunFunction(pVCpu);
1650	}
1651
1652	/* Avoid intercepting CR4 reads if the guest and shadow CR4 values are identical. */
1653	if (uShadowCr4 == pCtx->cr4)
1654	{
1655	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1656	pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(4);
1657	else
1658	{
1659	/* If the nested-hypervisor intercepts CR4 reads, we need to continue intercepting them. */
1660	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1661	pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx & ~RT_BIT(4))
1662	\| (pVmcbNstGstCache->u16InterceptRdCRx & RT_BIT(4));
1663	}
1664	}
1665	else
1666	pVmcb->ctrl.u16InterceptRdCRx \|= RT_BIT(4);
1667
1668	/* CR4 writes are always intercepted (both guest, nested-guest) for tracking PGM mode changes. */
1669	Assert(pVmcb->ctrl.u16InterceptWrCRx & RT_BIT(4));
1670
1671	/* Update VMCB with the shadow CR4 the appropriate VMCB clean bits. */
1672	Assert(!RT_HI_U32(uShadowCr4));
1673	pVmcb->guest.u64CR4 = uShadowCr4;
1674	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_CRX_EFER \| HMSVM_VMCB_CLEAN_INTERCEPTS);
1675
1676	return VINF_SUCCESS;
1677	}
1678
1679
1680	/**
1681	* Exports the guest (or nested-guest) control registers into the VMCB.
1682	*
1683	* @returns VBox status code.
1684	* @param pVCpu The cross context virtual CPU structure.
1685	* @param pVmcb Pointer to the VM control block.
1686	*
1687	* @remarks No-long-jump zone!!!
1688	*/
1689	static int hmR0SvmExportGuestControlRegs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1690	{
1691	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1692
1693	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR_MASK)
1694	{
1695	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR0)
1696	hmR0SvmExportGuestCR0(pVCpu, pVmcb);
1697
1698	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR2)
1699	{
1700	pVmcb->guest.u64CR2 = pVCpu->cpum.GstCtx.cr2;
1701	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CR2;
1702	}
1703
1704	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR3)
1705	hmR0SvmExportGuestCR3(pVCpu, pVmcb);
1706
1707	/* CR4 re-loading is ASSUMED to be done everytime we get in from ring-3! (XCR0) */
1708	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR4)
1709	{
1710	int rc = hmR0SvmExportGuestCR4(pVCpu, pVmcb);
1711	if (RT_FAILURE(rc))
1712	return rc;
1713	}
1714
1715	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_CR_MASK;
1716	}
1717	return VINF_SUCCESS;
1718	}
1719
1720
1721	/**
1722	* Exports the guest (or nested-guest) segment registers into the VMCB.
1723	*
1724	* @returns VBox status code.
1725	* @param pVCpu The cross context virtual CPU structure.
1726	* @param pVmcb Pointer to the VM control block.
1727	*
1728	* @remarks No-long-jump zone!!!
1729	*/
1730	static void hmR0SvmExportGuestSegmentRegs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1731	{
1732	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1733	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1734
1735	/* Guest segment registers. */
1736	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SREG_MASK)
1737	{
1738	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CS)
1739	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, CS, cs);
1740
1741	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SS)
1742	{
1743	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, SS, ss);
1744	pVmcb->guest.u8CPL = pCtx->ss.Attr.n.u2Dpl;
1745	}
1746
1747	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DS)
1748	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, DS, ds);
1749
1750	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_ES)
1751	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, ES, es);
1752
1753	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_FS)
1754	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, FS, fs);
1755
1756	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_GS)
1757	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, GS, gs);
1758
1759	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_SEG;
1760	}
1761
1762	/* Guest TR. */
1763	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_TR)
1764	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, TR, tr);
1765
1766	/* Guest LDTR. */
1767	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_LDTR)
1768	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, LDTR, ldtr);
1769
1770	/* Guest GDTR. */
1771	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_GDTR)
1772	{
1773	pVmcb->guest.GDTR.u32Limit = pCtx->gdtr.cbGdt;
1774	pVmcb->guest.GDTR.u64Base = pCtx->gdtr.pGdt;
1775	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1776	}
1777
1778	/* Guest IDTR. */
1779	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_IDTR)
1780	{
1781	pVmcb->guest.IDTR.u32Limit = pCtx->idtr.cbIdt;
1782	pVmcb->guest.IDTR.u64Base = pCtx->idtr.pIdt;
1783	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1784	}
1785
1786	pVCpu->hm.s.fCtxChanged &= ~( HM_CHANGED_GUEST_SREG_MASK
1787	\| HM_CHANGED_GUEST_TABLE_MASK);
1788	}
1789
1790
1791	/**
1792	* Exports the guest (or nested-guest) MSRs into the VMCB.
1793	*
1794	* @param pVCpu The cross context virtual CPU structure.
1795	* @param pVmcb Pointer to the VM control block.
1796	*
1797	* @remarks No-long-jump zone!!!
1798	*/
1799	static void hmR0SvmExportGuestMsrs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1800	{
1801	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1802	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1803
1804	/* Guest Sysenter MSRs. */
1805	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
1806	{
1807	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
1808	pVmcb->guest.u64SysEnterCS = pCtx->SysEnter.cs;
1809
1810	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
1811	pVmcb->guest.u64SysEnterEIP = pCtx->SysEnter.eip;
1812
1813	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
1814	pVmcb->guest.u64SysEnterESP = pCtx->SysEnter.esp;
1815	}
1816
1817	/*
1818	* Guest EFER MSR.
1819	* AMD-V requires guest EFER.SVME to be set. Weird.
1820	* See AMD spec. 15.5.1 "Basic Operation" \| "Canonicalization and Consistency Checks".
1821	*/
1822	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_EFER_MSR)
1823	{
1824	pVmcb->guest.u64EFER = pCtx->msrEFER \| MSR_K6_EFER_SVME;
1825	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1826	}
1827
1828	/* If the guest isn't in 64-bit mode, clear MSR_K6_LME bit, otherwise SVM expects amd64 shadow paging. */
1829	if ( !CPUMIsGuestInLongModeEx(pCtx)
1830	&& (pCtx->msrEFER & MSR_K6_EFER_LME))
1831	{
1832	pVmcb->guest.u64EFER &= ~MSR_K6_EFER_LME;
1833	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1834	}
1835
1836	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSCALL_MSRS)
1837	{
1838	pVmcb->guest.u64STAR = pCtx->msrSTAR;
1839	pVmcb->guest.u64LSTAR = pCtx->msrLSTAR;
1840	pVmcb->guest.u64CSTAR = pCtx->msrCSTAR;
1841	pVmcb->guest.u64SFMASK = pCtx->msrSFMASK;
1842	}
1843
1844	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_KERNEL_GS_BASE)
1845	pVmcb->guest.u64KernelGSBase = pCtx->msrKERNELGSBASE;
1846
1847	pVCpu->hm.s.fCtxChanged &= ~( HM_CHANGED_GUEST_SYSENTER_MSR_MASK
1848	\| HM_CHANGED_GUEST_EFER_MSR
1849	\| HM_CHANGED_GUEST_SYSCALL_MSRS
1850	\| HM_CHANGED_GUEST_KERNEL_GS_BASE);
1851
1852	/*
1853	* Setup the PAT MSR (applicable for Nested Paging only).
1854	*
1855	* The default value should be MSR_IA32_CR_PAT_INIT_VAL, but we treat all guest memory
1856	* as WB, so choose type 6 for all PAT slots, see @bugref{9634}.
1857	*
1858	* While guests can modify and see the modified values through the shadow values,
1859	* we shall not honor any guest modifications of this MSR to ensure caching is always
1860	* enabled similar to how we clear CR0.CD and NW bits.
1861	*
1862	* For nested-guests this needs to always be set as well, see @bugref{7243#c109}.
1863	*/
1864	pVmcb->guest.u64PAT = UINT64_C(0x0006060606060606);
1865
1866	/* Enable the last branch record bit if LBR virtualization is enabled. */
1867	if (pVmcb->ctrl.LbrVirt.n.u1LbrVirt)
1868	pVmcb->guest.u64DBGCTL = MSR_IA32_DEBUGCTL_LBR;
1869	}
1870
1871
1872	/**
1873	* Exports the guest (or nested-guest) debug state into the VMCB and programs
1874	* the necessary intercepts accordingly.
1875	*
1876	* @param pVCpu The cross context virtual CPU structure.
1877	* @param pVmcb Pointer to the VM control block.
1878	*
1879	* @remarks No-long-jump zone!!!
1880	* @remarks Requires EFLAGS to be up-to-date in the VMCB!
1881	*/
1882	static void hmR0SvmExportSharedDebugState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1883	{
1884	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1885
1886	/** @todo Figure out stepping with nested-guest. */
1887	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1888	{
1889	/*
1890	* We don't want to always intercept DRx read/writes for nested-guests as it causes
1891	* problems when the nested hypervisor isn't intercepting them, see @bugref{10080}.
1892	* Instead, they are strictly only requested when the nested hypervisor intercepts
1893	* them -- handled while merging VMCB controls.
1894	*
1895	* If neither the outer nor the nested-hypervisor is intercepting DRx read/writes,
1896	* then the nested-guest debug state should be actively loaded on the host so that
1897	* nested-guest reads/writes its own debug registers without causing VM-exits.
1898	*/
1899	if ( ( pVmcb->ctrl.u16InterceptRdDRx != 0xffff
1900	\|\| pVmcb->ctrl.u16InterceptWrDRx != 0xffff)
1901	&& !CPUMIsGuestDebugStateActive(pVCpu))
1902	{
1903	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
1904	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
1905	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
1906	Assert(CPUMIsGuestDebugStateActive(pVCpu));
1907	}
1908
1909	pVmcb->guest.u64DR6 = pCtx->dr[6];
1910	pVmcb->guest.u64DR7 = pCtx->dr[7];
1911	return;
1912	}
1913
1914	/*
1915	* Anyone single stepping on the host side? If so, we'll have to use the
1916	* trap flag in the guest EFLAGS since AMD-V doesn't have a trap flag on
1917	* the VMM level like the VT-x implementations does.
1918	*/
1919	bool fInterceptMovDRx = false;
1920	bool const fStepping = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
1921	if (fStepping)
1922	{
1923	pVCpu->hmr0.s.fClearTrapFlag = true;
1924	pVmcb->guest.u64RFlags \|= X86_EFL_TF;
1925	fInterceptMovDRx = true; /* Need clean DR6, no guest mess. */
1926	}
1927
1928	if ( fStepping
1929	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
1930	{
1931	/*
1932	* Use the combined guest and host DRx values found in the hypervisor
1933	* register set because the debugger has breakpoints active or someone
1934	* is single stepping on the host side.
1935	*
1936	* Note! DBGF expects a clean DR6 state before executing guest code.
1937	*/
1938	if (!CPUMIsHyperDebugStateActive(pVCpu))
1939	{
1940	CPUMR0LoadHyperDebugState(pVCpu, false /* include DR6 */);
1941	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
1942	Assert(CPUMIsHyperDebugStateActive(pVCpu));
1943	}
1944
1945	/* Update DR6 & DR7. (The other DRx values are handled by CPUM one way or the other.) */
1946	if ( pVmcb->guest.u64DR6 != X86_DR6_INIT_VAL
1947	\|\| pVmcb->guest.u64DR7 != CPUMGetHyperDR7(pVCpu))
1948	{
1949	pVmcb->guest.u64DR7 = CPUMGetHyperDR7(pVCpu);
1950	pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
1951	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1952	}
1953
1954	/** @todo If we cared, we could optimize to allow the guest to read registers
1955	* with the same values. */
1956	fInterceptMovDRx = true;
1957	pVCpu->hmr0.s.fUsingHyperDR7 = true;
1958	Log5(("hmR0SvmExportSharedDebugState: Loaded hyper DRx\n"));
1959	}
1960	else
1961	{
1962	/*
1963	* Update DR6, DR7 with the guest values if necessary.
1964	*/
1965	if ( pVmcb->guest.u64DR7 != pCtx->dr[7]
1966	\|\| pVmcb->guest.u64DR6 != pCtx->dr[6])
1967	{
1968	pVmcb->guest.u64DR7 = pCtx->dr[7];
1969	pVmcb->guest.u64DR6 = pCtx->dr[6];
1970	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1971	}
1972	pVCpu->hmr0.s.fUsingHyperDR7 = false;
1973
1974	/*
1975	* If the guest has enabled debug registers, we need to load them prior to
1976	* executing guest code so they'll trigger at the right time.
1977	*/
1978	if (pCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
1979	{
1980	if (!CPUMIsGuestDebugStateActive(pVCpu))
1981	{
1982	CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
1983	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
1984	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
1985	Assert(CPUMIsGuestDebugStateActive(pVCpu));
1986	}
1987	Log5(("hmR0SvmExportSharedDebugState: Loaded guest DRx\n"));
1988	}
1989	/*
1990	* If no debugging enabled, we'll lazy load DR0-3. We don't need to
1991	* intercept #DB as DR6 is updated in the VMCB.
1992	*
1993	* Note! If we cared and dared, we could skip intercepting \#DB here.
1994	* However, \#DB shouldn't be performance critical, so we'll play safe
1995	* and keep the code similar to the VT-x code and always intercept it.
1996	*/
1997	else if (!CPUMIsGuestDebugStateActive(pVCpu))
1998	fInterceptMovDRx = true;
1999	}
2000
2001	Assert(pVmcb->ctrl.u32InterceptXcpt & RT_BIT_32(X86_XCPT_DB));
2002	if (fInterceptMovDRx)
2003	{
2004	if ( pVmcb->ctrl.u16InterceptRdDRx != 0xffff
2005	\|\| pVmcb->ctrl.u16InterceptWrDRx != 0xffff)
2006	{
2007	pVmcb->ctrl.u16InterceptRdDRx = 0xffff;
2008	pVmcb->ctrl.u16InterceptWrDRx = 0xffff;
2009	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2010	}
2011	}
2012	else
2013	{
2014	if ( pVmcb->ctrl.u16InterceptRdDRx
2015	\|\| pVmcb->ctrl.u16InterceptWrDRx)
2016	{
2017	pVmcb->ctrl.u16InterceptRdDRx = 0;
2018	pVmcb->ctrl.u16InterceptWrDRx = 0;
2019	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2020	}
2021	}
2022	Log4Func(("DR6=%#RX64 DR7=%#RX64\n", pCtx->dr[6], pCtx->dr[7]));
2023	}
2024
2025	/**
2026	* Exports the hardware virtualization state into the nested-guest
2027	* VMCB.
2028	*
2029	* @param pVCpu The cross context virtual CPU structure.
2030	* @param pVmcb Pointer to the VM control block.
2031	*
2032	* @remarks No-long-jump zone!!!
2033	*/
2034	static void hmR0SvmExportGuestHwvirtState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2035	{
2036	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2037
2038	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_HWVIRT)
2039	{
2040	if (pVmcb->ctrl.IntCtrl.n.u1VGifEnable)
2041	{
2042	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2043	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
2044
2045	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx); /* Nested VGIF is not supported yet. */
2046	Assert(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VGIF); /* Physical hardware supports VGIF. */
2047	Assert(HMIsSvmVGifActive(pVM)); /* Outer VM has enabled VGIF. */
2048	NOREF(pVM);
2049
2050	pVmcb->ctrl.IntCtrl.n.u1VGif = CPUMGetGuestGif(pCtx);
2051	}
2052
2053	/*
2054	* Ensure the nested-guest pause-filter counters don't exceed the outer guest values esp.
2055	* since SVM doesn't have a preemption timer.
2056	*
2057	* We do this here rather than in hmR0SvmSetupVmcbNested() as we may have been executing the
2058	* nested-guest in IEM incl. PAUSE instructions which would update the pause-filter counters
2059	* and may continue execution in SVM R0 without a nested-guest #VMEXIT in between.
2060	*/
2061	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2062	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
2063	uint16_t const uGuestPauseFilterCount = pVM->hm.s.svm.cPauseFilter;
2064	uint16_t const uGuestPauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
2065	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, SVM_CTRL_INTERCEPT_PAUSE))
2066	{
2067	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2068	pVmcbCtrl->u16PauseFilterCount = RT_MIN(pCtx->hwvirt.svm.cPauseFilter, uGuestPauseFilterCount);
2069	pVmcbCtrl->u16PauseFilterThreshold = RT_MIN(pCtx->hwvirt.svm.cPauseFilterThreshold, uGuestPauseFilterThreshold);
2070	}
2071	else
2072	{
2073	/** @todo r=ramshankar: We can turn these assignments into assertions. */
2074	pVmcbCtrl->u16PauseFilterCount = uGuestPauseFilterCount;
2075	pVmcbCtrl->u16PauseFilterThreshold = uGuestPauseFilterThreshold;
2076	}
2077	pVmcbCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2078
2079	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_HWVIRT;
2080	}
2081	}
2082
2083
2084	/**
2085	* Exports the guest APIC TPR state into the VMCB.
2086	*
2087	* @returns VBox status code.
2088	* @param pVCpu The cross context virtual CPU structure.
2089	* @param pVmcb Pointer to the VM control block.
2090	*/
2091	static int hmR0SvmExportGuestApicTpr(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2092	{
2093	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2094
2095	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
2096	{
2097	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2098	if ( PDMHasApic(pVM)
2099	&& APICIsEnabled(pVCpu))
2100	{
2101	bool fPendingIntr;
2102	uint8_t u8Tpr;
2103	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, NULL /* pu8PendingIrq */);
2104	AssertRCReturn(rc, rc);
2105
2106	/* Assume that we need to trap all TPR accesses and thus need not check on
2107	every #VMEXIT if we should update the TPR. */
2108	Assert(pVmcb->ctrl.IntCtrl.n.u1VIntrMasking);
2109	pVCpu->hmr0.s.svm.fSyncVTpr = false;
2110
2111	if (!pVM->hm.s.fTprPatchingActive)
2112	{
2113	/* Bits 3-0 of the VTPR field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2114	pVmcb->ctrl.IntCtrl.n.u8VTPR = (u8Tpr >> 4);
2115
2116	/* If there are interrupts pending, intercept CR8 writes to evaluate ASAP if we
2117	can deliver the interrupt to the guest. */
2118	if (fPendingIntr)
2119	pVmcb->ctrl.u16InterceptWrCRx \|= RT_BIT(8);
2120	else
2121	{
2122	pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(8);
2123	pVCpu->hmr0.s.svm.fSyncVTpr = true;
2124	}
2125
2126	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_INTERCEPTS \| HMSVM_VMCB_CLEAN_INT_CTRL);
2127	}
2128	else
2129	{
2130	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2131	pVmcb->guest.u64LSTAR = u8Tpr;
2132	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hmr0.s.svm.pvMsrBitmap;
2133
2134	/* If there are interrupts pending, intercept LSTAR writes, otherwise don't intercept reads or writes. */
2135	if (fPendingIntr)
2136	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_INTERCEPT_WRITE);
2137	else
2138	{
2139	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
2140	pVCpu->hmr0.s.svm.fSyncVTpr = true;
2141	}
2142	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
2143	}
2144	}
2145	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
2146	}
2147	return VINF_SUCCESS;
2148	}
2149
2150
2151	/**
2152	* Sets up the exception interrupts required for guest execution in the VMCB.
2153	*
2154	* @param pVCpu The cross context virtual CPU structure.
2155	* @param pVmcb Pointer to the VM control block.
2156	*
2157	* @remarks No-long-jump zone!!!
2158	*/
2159	static void hmR0SvmExportGuestXcptIntercepts(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2160	{
2161	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2162
2163	/* If we modify intercepts from here, please check & adjust hmR0SvmMergeVmcbCtrlsNested() if required. */
2164	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_SVM_XCPT_INTERCEPTS)
2165	{
2166	/* Trap #UD for GIM provider (e.g. for hypercalls). */
2167	if (pVCpu->hm.s.fGIMTrapXcptUD \|\| pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit)
2168	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_UD);
2169	else
2170	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_UD);
2171
2172	/* Trap #BP for INT3 debug breakpoints set by the VM debugger. */
2173	if (pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
2174	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_BP);
2175	else
2176	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_BP);
2177
2178	/* The remaining intercepts are handled elsewhere, e.g. in hmR0SvmExportGuestCR0(). */
2179	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_SVM_XCPT_INTERCEPTS);
2180	}
2181	}
2182
2183
2184	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2185	/**
2186	* Merges guest and nested-guest intercepts for executing the nested-guest using
2187	* hardware-assisted SVM.
2188	*
2189	* This merges the guest and nested-guest intercepts in a way that if the outer
2190	* guest intercept is set we need to intercept it in the nested-guest as
2191	* well.
2192	*
2193	* @param pVCpu The cross context virtual CPU structure.
2194	* @param pVmcbNstGst Pointer to the nested-guest VM control block.
2195	*/
2196	static void hmR0SvmMergeVmcbCtrlsNested(PVMCPUCC pVCpu)
2197	{
2198	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2199	PCSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
2200	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2201	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2202
2203	/* Merge the guest's CR intercepts into the nested-guest VMCB. */
2204	pVmcbNstGstCtrl->u16InterceptRdCRx \|= pVmcb->ctrl.u16InterceptRdCRx;
2205	pVmcbNstGstCtrl->u16InterceptWrCRx \|= pVmcb->ctrl.u16InterceptWrCRx;
2206
2207	/* Always intercept CR4 writes for tracking PGM mode changes. */
2208	pVmcbNstGstCtrl->u16InterceptWrCRx \|= RT_BIT(4);
2209
2210	/* Without nested paging, intercept CR3 reads and writes as we load shadow page tables. */
2211	if (!pVM->hmr0.s.fNestedPaging)
2212	{
2213	pVmcbNstGstCtrl->u16InterceptRdCRx \|= RT_BIT(3);
2214	pVmcbNstGstCtrl->u16InterceptWrCRx \|= RT_BIT(3);
2215	}
2216
2217	/* Merge the guest's DR intercepts into the nested-guest VMCB. */
2218	pVmcbNstGstCtrl->u16InterceptRdDRx \|= pVmcb->ctrl.u16InterceptRdDRx;
2219	pVmcbNstGstCtrl->u16InterceptWrDRx \|= pVmcb->ctrl.u16InterceptWrDRx;
2220
2221	/*
2222	* Merge the guest's exception intercepts into the nested-guest VMCB.
2223	*
2224	* - #UD: Exclude these as the outer guest's GIM hypercalls are not applicable
2225	* while executing the nested-guest.
2226	*
2227	* - #BP: Exclude breakpoints set by the VM debugger for the outer guest. This can
2228	* be tweaked later depending on how we wish to implement breakpoints.
2229	*
2230	* - #GP: Exclude these as it's the inner VMMs problem to get vmsvga 3d drivers
2231	* loaded into their guests, not ours.
2232	*
2233	* Warning!! This ASSUMES we only intercept \#UD for hypercall purposes and \#BP
2234	* for VM debugger breakpoints, see hmR0SvmExportGuestXcptIntercepts().
2235	*/
2236	#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
2237	pVmcbNstGstCtrl->u32InterceptXcpt \|= pVmcb->ctrl.u32InterceptXcpt
2238	& ~( RT_BIT(X86_XCPT_UD)
2239	\| RT_BIT(X86_XCPT_BP)
2240	\| (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv ? RT_BIT(X86_XCPT_GP) : 0));
2241	#else
2242	pVmcbNstGstCtrl->u32InterceptXcpt \|= pVmcb->ctrl.u32InterceptXcpt;
2243	#endif
2244
2245	/*
2246	* Adjust intercepts while executing the nested-guest that differ from the
2247	* outer guest intercepts.
2248	*
2249	* - VINTR: Exclude the outer guest intercept as we don't need to cause VINTR #VMEXITs
2250	* that belong to the nested-guest to the outer guest.
2251	*
2252	* - VMMCALL: Exclude the outer guest intercept as when it's also not intercepted by
2253	* the nested-guest, the physical CPU raises a \#UD exception as expected.
2254	*/
2255	pVmcbNstGstCtrl->u64InterceptCtrl \|= (pVmcb->ctrl.u64InterceptCtrl & ~( SVM_CTRL_INTERCEPT_VINTR
2256	\| SVM_CTRL_INTERCEPT_VMMCALL))
2257	\| HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS;
2258
2259	Assert( (pVmcbNstGstCtrl->u64InterceptCtrl & HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS)
2260	== HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS);
2261
2262	/* Finally, update the VMCB clean bits. */
2263	pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2264	}
2265	#endif
2266
2267
2268	/**
2269	* Enters the AMD-V session.
2270	*
2271	* @returns VBox status code.
2272	* @param pVCpu The cross context virtual CPU structure.
2273	*/
2274	VMMR0DECL(int) SVMR0Enter(PVMCPUCC pVCpu)
2275	{
2276	AssertPtr(pVCpu);
2277	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fSupported);
2278	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2279
2280	LogFlowFunc(("pVCpu=%p\n", pVCpu));
2281	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
2282	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
2283
2284	pVCpu->hmr0.s.fLeaveDone = false;
2285	return VINF_SUCCESS;
2286	}
2287
2288
2289	/**
2290	* Thread-context callback for AMD-V.
2291	*
2292	* This is used together with RTThreadCtxHookCreate() on platforms which
2293	* supports it, and directly from VMMR0EmtPrepareForBlocking() and
2294	* VMMR0EmtResumeAfterBlocking() on platforms which don't.
2295	*
2296	* @param enmEvent The thread-context event.
2297	* @param pVCpu The cross context virtual CPU structure.
2298	* @param fGlobalInit Whether global VT-x/AMD-V init. is used.
2299	* @thread EMT(pVCpu)
2300	*/
2301	VMMR0DECL(void) SVMR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPUCC pVCpu, bool fGlobalInit)
2302	{
2303	NOREF(fGlobalInit);
2304
2305	switch (enmEvent)
2306	{
2307	case RTTHREADCTXEVENT_OUT:
2308	{
2309	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2310	VMCPU_ASSERT_EMT(pVCpu);
2311
2312	/* No longjmps (log-flush, locks) in this fragile context. */
2313	VMMRZCallRing3Disable(pVCpu);
2314
2315	if (!pVCpu->hmr0.s.fLeaveDone)
2316	{
2317	hmR0SvmLeave(pVCpu, false /* fImportState */);
2318	pVCpu->hmr0.s.fLeaveDone = true;
2319	}
2320
2321	/* Leave HM context, takes care of local init (term). */
2322	int rc = HMR0LeaveCpu(pVCpu);
2323	AssertRC(rc); NOREF(rc);
2324
2325	/* Restore longjmp state. */
2326	VMMRZCallRing3Enable(pVCpu);
2327	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
2328	break;
2329	}
2330
2331	case RTTHREADCTXEVENT_IN:
2332	{
2333	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2334	VMCPU_ASSERT_EMT(pVCpu);
2335
2336	/* No longjmps (log-flush, locks) in this fragile context. */
2337	VMMRZCallRing3Disable(pVCpu);
2338
2339	/*
2340	* Initialize the bare minimum state required for HM. This takes care of
2341	* initializing AMD-V if necessary (onlined CPUs, local init etc.)
2342	*/
2343	int rc = hmR0EnterCpu(pVCpu);
2344	AssertRC(rc); NOREF(rc);
2345	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
2346	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
2347
2348	pVCpu->hmr0.s.fLeaveDone = false;
2349
2350	/* Restore longjmp state. */
2351	VMMRZCallRing3Enable(pVCpu);
2352	break;
2353	}
2354
2355	default:
2356	break;
2357	}
2358	}
2359
2360
2361	/**
2362	* Saves the host state.
2363	*
2364	* @returns VBox status code.
2365	* @param pVCpu The cross context virtual CPU structure.
2366	*
2367	* @remarks No-long-jump zone!!!
2368	*/
2369	VMMR0DECL(int) SVMR0ExportHostState(PVMCPUCC pVCpu)
2370	{
2371	NOREF(pVCpu);
2372
2373	/* Nothing to do here. AMD-V does this for us automatically during the world-switch. */
2374	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_HOST_CONTEXT);
2375	return VINF_SUCCESS;
2376	}
2377
2378
2379	/**
2380	* Exports the guest or nested-guest state from the virtual-CPU context into the
2381	* VMCB.
2382	*
2383	* Also sets up the appropriate VMRUN function to execute guest or nested-guest
2384	* code based on the virtual-CPU mode.
2385	*
2386	* @returns VBox status code.
2387	* @param pVCpu The cross context virtual CPU structure.
2388	* @param pSvmTransient Pointer to the SVM-transient structure.
2389	*
2390	* @remarks No-long-jump zone!!!
2391	*/
2392	static int hmR0SvmExportGuestState(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient)
2393	{
2394	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
2395
2396	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
2397	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2398	Assert(pVmcb);
2399
2400	pVmcb->guest.u64RIP = pCtx->rip;
2401	pVmcb->guest.u64RSP = pCtx->rsp;
2402	pVmcb->guest.u64RFlags = pCtx->eflags.u32;
2403	pVmcb->guest.u64RAX = pCtx->rax;
2404
2405	bool const fIsNestedGuest = pSvmTransient->fIsNestedGuest;
2406	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
2407
2408	int rc = hmR0SvmExportGuestControlRegs(pVCpu, pVmcb);
2409	AssertRCReturnStmt(rc, ASMSetFlags(fEFlags), rc);
2410	hmR0SvmExportGuestSegmentRegs(pVCpu, pVmcb);
2411	hmR0SvmExportGuestMsrs(pVCpu, pVmcb);
2412	hmR0SvmExportGuestHwvirtState(pVCpu, pVmcb);
2413
2414	ASMSetFlags(fEFlags);
2415
2416	if (!fIsNestedGuest)
2417	{
2418	/* hmR0SvmExportGuestApicTpr() must be called -after- hmR0SvmExportGuestMsrs() as we
2419	otherwise we would overwrite the LSTAR MSR that we use for TPR patching. */
2420	hmR0SvmExportGuestApicTpr(pVCpu, pVmcb);
2421	hmR0SvmExportGuestXcptIntercepts(pVCpu, pVmcb);
2422	}
2423
2424	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
2425	uint64_t fUnusedMask = HM_CHANGED_GUEST_RIP
2426	\| HM_CHANGED_GUEST_RFLAGS
2427	\| HM_CHANGED_GUEST_GPRS_MASK
2428	\| HM_CHANGED_GUEST_X87
2429	\| HM_CHANGED_GUEST_SSE_AVX
2430	\| HM_CHANGED_GUEST_OTHER_XSAVE
2431	\| HM_CHANGED_GUEST_XCRx
2432	\| HM_CHANGED_GUEST_TSC_AUX
2433	\| HM_CHANGED_GUEST_OTHER_MSRS;
2434	if (fIsNestedGuest)
2435	fUnusedMask \|= HM_CHANGED_SVM_XCPT_INTERCEPTS
2436	\| HM_CHANGED_GUEST_APIC_TPR;
2437
2438	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( fUnusedMask
2439	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_SVM_MASK)));
2440
2441	#ifdef VBOX_STRICT
2442	/*
2443	* All of the guest-CPU state and SVM keeper bits should be exported here by now,
2444	* except for the host-context and/or shared host-guest context bits.
2445	*/
2446	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
2447	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)),
2448	("fCtxChanged=%#RX64\n", fCtxChanged));
2449
2450	/*
2451	* If we need to log state that isn't always imported, we'll need to import them here.
2452	* See hmR0SvmPostRunGuest() for which part of the state is imported uncondtionally.
2453	*/
2454	hmR0SvmLogState(pVCpu, pVmcb, "hmR0SvmExportGuestState", 0 /* fFlags /, 0 / uVerbose */);
2455	#endif
2456
2457	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
2458	return VINF_SUCCESS;
2459	}
2460
2461	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2462
2463	/**
2464	* Merges the guest and nested-guest MSR permission bitmap.
2465	*
2466	* If the guest is intercepting an MSR we need to intercept it regardless of
2467	* whether the nested-guest is intercepting it or not.
2468	*
2469	* @param pHostCpu The HM physical-CPU structure.
2470	* @param pVCpu The cross context virtual CPU structure.
2471	*
2472	* @remarks No-long-jmp zone!!!
2473	*/
2474	DECLINLINE(void) hmR0SvmMergeMsrpmNested(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu)
2475	{
2476	uint64_t const pu64GstMsrpm = (uint64_t const )pVCpu->hmr0.s.svm.pvMsrBitmap;
2477	uint64_t const pu64NstGstMsrpm = (uint64_t const )&pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[0];
2478	uint64_t pu64DstMsrpm = (uint64_t )pHostCpu->n.svm.pvNstGstMsrpm;
2479
2480	/* MSRPM bytes from offset 0x1800 are reserved, so we stop merging there. */
2481	uint32_t const offRsvdQwords = 0x1800 >> 3;
2482	for (uint32_t i = 0; i < offRsvdQwords; i++)
2483	pu64DstMsrpm[i] = pu64NstGstMsrpm[i] \| pu64GstMsrpm[i];
2484	}
2485
2486
2487	/**
2488	* Caches the nested-guest VMCB fields before we modify them for execution using
2489	* hardware-assisted SVM.
2490	*
2491	* @returns true if the VMCB was previously already cached, false otherwise.
2492	* @param pVCpu The cross context virtual CPU structure.
2493	*
2494	* @sa HMNotifySvmNstGstVmexit.
2495	*/
2496	static bool hmR0SvmCacheVmcbNested(PVMCPUCC pVCpu)
2497	{
2498	/*
2499	* Cache the nested-guest programmed VMCB fields if we have not cached it yet.
2500	* Otherwise we risk re-caching the values we may have modified, see @bugref{7243#c44}.
2501	*
2502	* Nested-paging CR3 is not saved back into the VMCB on #VMEXIT, hence no need to
2503	* cache and restore it, see AMD spec. 15.25.4 "Nested Paging and VMRUN/#VMEXIT".
2504	*/
2505	PSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
2506	bool const fWasCached = pVmcbNstGstCache->fCacheValid;
2507	if (!fWasCached)
2508	{
2509	PCSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2510	PCSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2511	pVmcbNstGstCache->u16InterceptRdCRx = pVmcbNstGstCtrl->u16InterceptRdCRx;
2512	pVmcbNstGstCache->u16InterceptWrCRx = pVmcbNstGstCtrl->u16InterceptWrCRx;
2513	pVmcbNstGstCache->u16InterceptRdDRx = pVmcbNstGstCtrl->u16InterceptRdDRx;
2514	pVmcbNstGstCache->u16InterceptWrDRx = pVmcbNstGstCtrl->u16InterceptWrDRx;
2515	pVmcbNstGstCache->u16PauseFilterThreshold = pVmcbNstGstCtrl->u16PauseFilterThreshold;
2516	pVmcbNstGstCache->u16PauseFilterCount = pVmcbNstGstCtrl->u16PauseFilterCount;
2517	pVmcbNstGstCache->u32InterceptXcpt = pVmcbNstGstCtrl->u32InterceptXcpt;
2518	pVmcbNstGstCache->u64InterceptCtrl = pVmcbNstGstCtrl->u64InterceptCtrl;
2519	pVmcbNstGstCache->u64TSCOffset = pVmcbNstGstCtrl->u64TSCOffset;
2520	pVmcbNstGstCache->fVIntrMasking = pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking;
2521	pVmcbNstGstCache->fNestedPaging = pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging;
2522	pVmcbNstGstCache->fLbrVirt = pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt;
2523	pVmcbNstGstCache->fCacheValid = true;
2524	Log4Func(("Cached VMCB fields\n"));
2525	}
2526
2527	return fWasCached;
2528	}
2529
2530
2531	/**
2532	* Sets up the nested-guest VMCB for execution using hardware-assisted SVM.
2533	*
2534	* This is done the first time we enter nested-guest execution using SVM R0
2535	* until the nested-guest \#VMEXIT (not to be confused with physical CPU
2536	* \#VMEXITs which may or may not cause a corresponding nested-guest \#VMEXIT).
2537	*
2538	* @param pVCpu The cross context virtual CPU structure.
2539	*/
2540	static void hmR0SvmSetupVmcbNested(PVMCPUCC pVCpu)
2541	{
2542	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2543	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2544
2545	HMSVM_ASSERT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2546
2547	/*
2548	* First cache the nested-guest VMCB fields we may potentially modify.
2549	*/
2550	bool const fVmcbCached = hmR0SvmCacheVmcbNested(pVCpu);
2551	if (!fVmcbCached)
2552	{
2553	/*
2554	* The IOPM of the nested-guest can be ignored because the the guest always
2555	* intercepts all IO port accesses. Thus, we'll swap to the guest IOPM rather
2556	* than the nested-guest IOPM and swap the field back on the #VMEXIT.
2557	*/
2558	pVmcbNstGstCtrl->u64IOPMPhysAddr = g_HCPhysIOBitmap;
2559
2560	/*
2561	* Use the same nested-paging as the outer guest. We can't dynamically switch off
2562	* nested-paging suddenly while executing a VM (see assertion at the end of
2563	* Trap0eHandler() in PGMAllBth.h).
2564	*/
2565	pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging = pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging;
2566
2567	/* Always enable V_INTR_MASKING as we do not want to allow access to the physical APIC TPR. */
2568	pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking = 1;
2569
2570	/*
2571	* Turn off TPR syncing on #VMEXIT for nested-guests as CR8 intercepts are subject
2572	* to the nested-guest intercepts and we always run with V_INTR_MASKING.
2573	*/
2574	pVCpu->hmr0.s.svm.fSyncVTpr = false;
2575
2576	# ifdef DEBUG_ramshankar
2577	/* For debugging purposes - copy the LBR info. from outer guest VMCB. */
2578	pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt = pVmcb->ctrl.LbrVirt.n.u1LbrVirt;
2579	# endif
2580
2581	/*
2582	* If we don't expose Virtualized-VMSAVE/VMLOAD feature to the outer guest, we
2583	* need to intercept VMSAVE/VMLOAD instructions executed by the nested-guest.
2584	*/
2585	if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVirtVmsaveVmload)
2586	pVmcbNstGstCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_VMSAVE
2587	\| SVM_CTRL_INTERCEPT_VMLOAD;
2588
2589	/*
2590	* If we don't expose Virtual GIF feature to the outer guest, we need to intercept
2591	* CLGI/STGI instructions executed by the nested-guest.
2592	*/
2593	if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVGif)
2594	pVmcbNstGstCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_CLGI
2595	\| SVM_CTRL_INTERCEPT_STGI;
2596
2597	/* Merge the guest and nested-guest intercepts. */
2598	hmR0SvmMergeVmcbCtrlsNested(pVCpu);
2599
2600	/* Update the VMCB clean bits. */
2601	pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2602	}
2603	else
2604	{
2605	Assert(!pVCpu->hmr0.s.svm.fSyncVTpr);
2606	Assert(pVmcbNstGstCtrl->u64IOPMPhysAddr == g_HCPhysIOBitmap);
2607	Assert(RT_BOOL(pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging) == pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
2608	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPagingCfg == pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
2609	}
2610	}
2611
2612	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
2613
2614	/**
2615	* Exports the state shared between the host and guest (or nested-guest) into
2616	* the VMCB.
2617	*
2618	* @param pVCpu The cross context virtual CPU structure.
2619	* @param pVmcb Pointer to the VM control block.
2620	*
2621	* @remarks No-long-jump zone!!!
2622	*/
2623	static void hmR0SvmExportSharedState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2624	{
2625	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2626	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2627
2628	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
2629	hmR0SvmExportSharedDebugState(pVCpu, pVmcb);
2630
2631	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
2632	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE),
2633	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
2634	}
2635
2636
2637	/**
2638	* Worker for SVMR0ImportStateOnDemand.
2639	*
2640	* @param pVCpu The cross context virtual CPU structure.
2641	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
2642	*/
2643	static void hmR0SvmImportGuestState(PVMCPUCC pVCpu, uint64_t fWhat)
2644	{
2645	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
2646
2647	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2648	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
2649	PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
2650	PCSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
2651
2652	/*
2653	* We disable interrupts to make the updating of the state and in particular
2654	* the fExtrn modification atomic wrt to preemption hooks.
2655	*/
2656	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
2657
2658	fWhat &= pCtx->fExtrn;
2659	if (fWhat)
2660	{
2661	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2662	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
2663	{
2664	if (pVmcbCtrl->IntCtrl.n.u1VGifEnable)
2665	{
2666	Assert(!CPUMIsGuestInSvmNestedHwVirtMode(pCtx)); /* We don't yet support passing VGIF feature to the guest. */
2667	Assert(HMIsSvmVGifActive(pVCpu->CTX_SUFF(pVM))); /* VM has configured it. */
2668	CPUMSetGuestGif(pCtx, pVmcbCtrl->IntCtrl.n.u1VGif);
2669	}
2670	}
2671
2672	if (fWhat & CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ)
2673	{
2674	if ( !pVmcbCtrl->IntCtrl.n.u1VIrqPending
2675	&& VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST))
2676	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST);
2677	}
2678	#endif
2679
2680	if (fWhat & CPUMCTX_EXTRN_INHIBIT_INT)
2681	{
2682	if (pVmcbCtrl->IntShadow.n.u1IntShadow)
2683	EMSetInhibitInterruptsPC(pVCpu, pVmcbGuest->u64RIP);
2684	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2685	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2686	}
2687
2688	if (fWhat & CPUMCTX_EXTRN_RIP)
2689	pCtx->rip = pVmcbGuest->u64RIP;
2690
2691	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
2692	{
2693	pCtx->eflags.u32 = pVmcbGuest->u64RFlags;
2694	if (pVCpu->hmr0.s.fClearTrapFlag)
2695	{
2696	pVCpu->hmr0.s.fClearTrapFlag = false;
2697	pCtx->eflags.Bits.u1TF = 0;
2698	}
2699	}
2700
2701	if (fWhat & CPUMCTX_EXTRN_RSP)
2702	pCtx->rsp = pVmcbGuest->u64RSP;
2703
2704	if (fWhat & CPUMCTX_EXTRN_RAX)
2705	pCtx->rax = pVmcbGuest->u64RAX;
2706
2707	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
2708	{
2709	if (fWhat & CPUMCTX_EXTRN_CS)
2710	{
2711	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, CS, cs);
2712	/* Correct the CS granularity bit. Haven't seen it being wrong in any other register (yet). */
2713	/** @todo SELM might need to be fixed as it too should not care about the
2714	* granularity bit. See @bugref{6785}. */
2715	if ( !pCtx->cs.Attr.n.u1Granularity
2716	&& pCtx->cs.Attr.n.u1Present
2717	&& pCtx->cs.u32Limit > UINT32_C(0xfffff))
2718	{
2719	Assert((pCtx->cs.u32Limit & 0xfff) == 0xfff);
2720	pCtx->cs.Attr.n.u1Granularity = 1;
2721	}
2722	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, cs);
2723	}
2724	if (fWhat & CPUMCTX_EXTRN_SS)
2725	{
2726	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, SS, ss);
2727	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, ss);
2728	/*
2729	* Sync the hidden SS DPL field. AMD CPUs have a separate CPL field in the
2730	* VMCB and uses that and thus it's possible that when the CPL changes during
2731	* guest execution that the SS DPL isn't updated by AMD-V. Observed on some
2732	* AMD Fusion CPUs with 64-bit guests.
2733	*
2734	* See AMD spec. 15.5.1 "Basic operation".
2735	*/
2736	Assert(!(pVmcbGuest->u8CPL & ~0x3));
2737	uint8_t const uCpl = pVmcbGuest->u8CPL;
2738	if (pCtx->ss.Attr.n.u2Dpl != uCpl)
2739	pCtx->ss.Attr.n.u2Dpl = uCpl & 0x3;
2740	}
2741	if (fWhat & CPUMCTX_EXTRN_DS)
2742	{
2743	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, DS, ds);
2744	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, ds);
2745	}
2746	if (fWhat & CPUMCTX_EXTRN_ES)
2747	{
2748	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, ES, es);
2749	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, es);
2750	}
2751	if (fWhat & CPUMCTX_EXTRN_FS)
2752	{
2753	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, FS, fs);
2754	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, fs);
2755	}
2756	if (fWhat & CPUMCTX_EXTRN_GS)
2757	{
2758	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, GS, gs);
2759	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, gs);
2760	}
2761	}
2762
2763	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
2764	{
2765	if (fWhat & CPUMCTX_EXTRN_TR)
2766	{
2767	/*
2768	* Fixup TR attributes so it's compatible with Intel. Important when saved-states
2769	* are used between Intel and AMD, see @bugref{6208#c39}.
2770	* ASSUME that it's normally correct and that we're in 32-bit or 64-bit mode.
2771	*/
2772	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, TR, tr);
2773	if (pCtx->tr.Attr.n.u4Type != X86_SEL_TYPE_SYS_386_TSS_BUSY)
2774	{
2775	if ( pCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL
2776	\|\| CPUMIsGuestInLongModeEx(pCtx))
2777	pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
2778	else if (pCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
2779	pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_286_TSS_BUSY;
2780	}
2781	}
2782
2783	if (fWhat & CPUMCTX_EXTRN_LDTR)
2784	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, LDTR, ldtr);
2785
2786	if (fWhat & CPUMCTX_EXTRN_GDTR)
2787	{
2788	pCtx->gdtr.cbGdt = pVmcbGuest->GDTR.u32Limit;
2789	pCtx->gdtr.pGdt = pVmcbGuest->GDTR.u64Base;
2790	}
2791
2792	if (fWhat & CPUMCTX_EXTRN_IDTR)
2793	{
2794	pCtx->idtr.cbIdt = pVmcbGuest->IDTR.u32Limit;
2795	pCtx->idtr.pIdt = pVmcbGuest->IDTR.u64Base;
2796	}
2797	}
2798
2799	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
2800	{
2801	pCtx->msrSTAR = pVmcbGuest->u64STAR;
2802	pCtx->msrLSTAR = pVmcbGuest->u64LSTAR;
2803	pCtx->msrCSTAR = pVmcbGuest->u64CSTAR;
2804	pCtx->msrSFMASK = pVmcbGuest->u64SFMASK;
2805	}
2806
2807	if ( (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
2808	&& !pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit /* Intercepted. AMD-V would clear the high 32 bits of EIP & ESP. */)
2809	{
2810	pCtx->SysEnter.cs = pVmcbGuest->u64SysEnterCS;
2811	pCtx->SysEnter.eip = pVmcbGuest->u64SysEnterEIP;
2812	pCtx->SysEnter.esp = pVmcbGuest->u64SysEnterESP;
2813	}
2814
2815	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
2816	pCtx->msrKERNELGSBASE = pVmcbGuest->u64KernelGSBase;
2817
2818	if (fWhat & CPUMCTX_EXTRN_DR_MASK)
2819	{
2820	if (fWhat & CPUMCTX_EXTRN_DR6)
2821	{
2822	if (!pVCpu->hmr0.s.fUsingHyperDR7)
2823	pCtx->dr[6] = pVmcbGuest->u64DR6;
2824	else
2825	CPUMSetHyperDR6(pVCpu, pVmcbGuest->u64DR6);
2826	}
2827
2828	if (fWhat & CPUMCTX_EXTRN_DR7)
2829	{
2830	if (!pVCpu->hmr0.s.fUsingHyperDR7)
2831	pCtx->dr[7] = pVmcbGuest->u64DR7;
2832	else
2833	Assert(pVmcbGuest->u64DR7 == CPUMGetHyperDR7(pVCpu));
2834	}
2835	}
2836
2837	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
2838	{
2839	if (fWhat & CPUMCTX_EXTRN_CR0)
2840	{
2841	/* We intercept changes to all CR0 bits except maybe TS & MP bits. */
2842	uint64_t const uCr0 = (pCtx->cr0 & ~(X86_CR0_TS \| X86_CR0_MP))
2843	\| (pVmcbGuest->u64CR0 & (X86_CR0_TS \| X86_CR0_MP));
2844	VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
2845	CPUMSetGuestCR0(pVCpu, uCr0);
2846	VMMRZCallRing3Enable(pVCpu);
2847	}
2848
2849	if (fWhat & CPUMCTX_EXTRN_CR2)
2850	pCtx->cr2 = pVmcbGuest->u64CR2;
2851
2852	if (fWhat & CPUMCTX_EXTRN_CR3)
2853	{
2854	if ( pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging
2855	&& pCtx->cr3 != pVmcbGuest->u64CR3)
2856	{
2857	CPUMSetGuestCR3(pVCpu, pVmcbGuest->u64CR3);
2858	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
2859	}
2860	}
2861
2862	/* Changes to CR4 are always intercepted. */
2863	}
2864
2865	/* Update fExtrn. */
2866	pCtx->fExtrn &= ~fWhat;
2867
2868	/* If everything has been imported, clear the HM keeper bit. */
2869	if (!(pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL))
2870	{
2871	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
2872	Assert(!pCtx->fExtrn);
2873	}
2874	}
2875	else
2876	Assert(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
2877
2878	ASMSetFlags(fEFlags);
2879
2880	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
2881
2882	/*
2883	* Honor any pending CR3 updates.
2884	*
2885	* Consider this scenario: #VMEXIT -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp
2886	* -> SVMR0CallRing3Callback() -> VMMRZCallRing3Disable() -> hmR0SvmImportGuestState()
2887	* -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp -> continue with #VMEXIT
2888	* handling -> hmR0SvmImportGuestState() and here we are.
2889	*
2890	* The reason for such complicated handling is because VM-exits that call into PGM expect
2891	* CR3 to be up-to-date and thus any CR3-saves -before- the VM-exit (longjmp) would've
2892	* postponed the CR3 update via the force-flag and cleared CR3 from fExtrn. Any SVM R0
2893	* VM-exit handler that requests CR3 to be saved will end up here and we call PGMUpdateCR3().
2894	*
2895	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again,
2896	* and does not process force-flag like regular exits to ring-3 either, we cover for it here.
2897	*/
2898	if ( VMMRZCallRing3IsEnabled(pVCpu)
2899	&& VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
2900	{
2901	AssertMsg(pCtx->cr3 == pVmcbGuest->u64CR3, ("cr3=%#RX64 vmcb_cr3=%#RX64\n", pCtx->cr3, pVmcbGuest->u64CR3));
2902	PGMUpdateCR3(pVCpu, pCtx->cr3);
2903	}
2904	}
2905
2906
2907	/**
2908	* Saves the guest (or nested-guest) state from the VMCB into the guest-CPU
2909	* context.
2910	*
2911	* Currently there is no residual state left in the CPU that is not updated in the
2912	* VMCB.
2913	*
2914	* @returns VBox status code.
2915	* @param pVCpu The cross context virtual CPU structure.
2916	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
2917	*/
2918	VMMR0DECL(int) SVMR0ImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
2919	{
2920	hmR0SvmImportGuestState(pVCpu, fWhat);
2921	return VINF_SUCCESS;
2922	}
2923
2924
2925	/**
2926	* Gets SVM \#VMEXIT auxiliary information.
2927	*
2928	* @returns VBox status code.
2929	* @param pVCpu The cross context virtual CPU structure.
2930	* @param pSvmExitAux Where to store the auxiliary info.
2931	*/
2932	VMMR0DECL(int) SVMR0GetExitAuxInfo(PVMCPUCC pVCpu, PSVMEXITAUX pSvmExitAux)
2933	{
2934	PCSVMTRANSIENT pSvmTransient = pVCpu->hmr0.s.svm.pSvmTransient;
2935	if (RT_LIKELY(pSvmTransient))
2936	{
2937	PCSVMVMCB pVmcb = pSvmTransient->pVmcb;
2938	if (RT_LIKELY(pVmcb))
2939	{
2940	pSvmExitAux->u64ExitCode = pVmcb->ctrl.u64ExitCode;
2941	pSvmExitAux->u64ExitInfo1 = pVmcb->ctrl.u64ExitInfo1;
2942	pSvmExitAux->u64ExitInfo2 = pVmcb->ctrl.u64ExitInfo2;
2943	pSvmExitAux->ExitIntInfo = pVmcb->ctrl.ExitIntInfo;
2944	return VINF_SUCCESS;
2945	}
2946	return VERR_SVM_IPE_5;
2947	}
2948	return VERR_NOT_AVAILABLE;
2949	}
2950
2951
2952	/**
2953	* Does the necessary state syncing before returning to ring-3 for any reason
2954	* (longjmp, preemption, voluntary exits to ring-3) from AMD-V.
2955	*
2956	* @param pVCpu The cross context virtual CPU structure.
2957	* @param fImportState Whether to import the guest state from the VMCB back
2958	* to the guest-CPU context.
2959	*
2960	* @remarks No-long-jmp zone!!!
2961	*/
2962	static void hmR0SvmLeave(PVMCPUCC pVCpu, bool fImportState)
2963	{
2964	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2965	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2966
2967	/*
2968	* !!! IMPORTANT !!!
2969	* If you modify code here, make sure to check whether SVMR0CallRing3Callback() needs to be updated too.
2970	*/
2971
2972	/* Save the guest state if necessary. */
2973	if (fImportState)
2974	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
2975
2976	/* Restore host FPU state if necessary and resync on next R0 reentry. */
2977	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
2978	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
2979
2980	/*
2981	* Restore host debug registers if necessary and resync on next R0 reentry.
2982	*/
2983	#ifdef VBOX_STRICT
2984	if (CPUMIsHyperDebugStateActive(pVCpu))
2985	{
2986	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb; /** @todo nested-guest. */
2987	Assert(pVmcb->ctrl.u16InterceptRdDRx == 0xffff);
2988	Assert(pVmcb->ctrl.u16InterceptWrDRx == 0xffff);
2989	}
2990	#endif
2991	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
2992	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
2993	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
2994
2995	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
2996	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
2997	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
2998	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
2999	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
3000	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitVmentry);
3001	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
3002
3003	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
3004	}
3005
3006
3007	/**
3008	* Leaves the AMD-V session.
3009	*
3010	* Only used while returning to ring-3 either due to longjump or exits to
3011	* ring-3.
3012	*
3013	* @returns VBox status code.
3014	* @param pVCpu The cross context virtual CPU structure.
3015	*/
3016	static int hmR0SvmLeaveSession(PVMCPUCC pVCpu)
3017	{
3018	HM_DISABLE_PREEMPT(pVCpu);
3019	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3020	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3021
3022	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
3023	and done this from the SVMR0ThreadCtxCallback(). */
3024	if (!pVCpu->hmr0.s.fLeaveDone)
3025	{
3026	hmR0SvmLeave(pVCpu, true /* fImportState */);
3027	pVCpu->hmr0.s.fLeaveDone = true;
3028	}
3029
3030	/*
3031	* !!! IMPORTANT !!!
3032	* If you modify code here, make sure to check whether SVMR0CallRing3Callback() needs to be updated too.
3033	*/
3034
3035	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
3036	/* Deregister hook now that we've left HM context before re-enabling preemption. */
3037	VMMR0ThreadCtxHookDisable(pVCpu);
3038
3039	/* Leave HM context. This takes care of local init (term). */
3040	int rc = HMR0LeaveCpu(pVCpu);
3041
3042	HM_RESTORE_PREEMPT();
3043	return rc;
3044	}
3045
3046
3047	/**
3048	* VMMRZCallRing3() callback wrapper which saves the guest state (or restores
3049	* any remaining host state) before we go back to ring-3 due to an assertion.
3050	*
3051	* @param pVCpu The cross context virtual CPU structure.
3052	*/
3053	VMMR0DECL(int) SVMR0AssertionCallback(PVMCPUCC pVCpu)
3054	{
3055	/*
3056	* !!! IMPORTANT !!!
3057	* If you modify code here, make sure to check whether hmR0SvmLeave() and hmR0SvmLeaveSession() needs
3058	* to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
3059	*/
3060	VMMR0AssertionRemoveNotification(pVCpu);
3061	VMMRZCallRing3Disable(pVCpu);
3062	HM_DISABLE_PREEMPT(pVCpu);
3063
3064	/* Import the entire guest state. */
3065	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3066
3067	/* Restore host FPU state if necessary and resync on next R0 reentry. */
3068	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
3069
3070	/* Restore host debug registers if necessary and resync on next R0 reentry. */
3071	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
3072
3073	/* Deregister the hook now that we've left HM context before re-enabling preemption. */
3074	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
3075	VMMR0ThreadCtxHookDisable(pVCpu);
3076
3077	/* Leave HM context. This takes care of local init (term). */
3078	HMR0LeaveCpu(pVCpu);
3079
3080	HM_RESTORE_PREEMPT();
3081	return VINF_SUCCESS;
3082	}
3083
3084
3085	/**
3086	* Take necessary actions before going back to ring-3.
3087	*
3088	* An action requires us to go back to ring-3. This function does the necessary
3089	* steps before we can safely return to ring-3. This is not the same as longjmps
3090	* to ring-3, this is voluntary.
3091	*
3092	* @returns Strict VBox status code.
3093	* @param pVCpu The cross context virtual CPU structure.
3094	* @param rcExit The reason for exiting to ring-3. Can be
3095	* VINF_VMM_UNKNOWN_RING3_CALL.
3096	*/
3097	static VBOXSTRICTRC hmR0SvmExitToRing3(PVMCPUCC pVCpu, VBOXSTRICTRC rcExit)
3098	{
3099	Assert(pVCpu);
3100	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
3101
3102	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
3103	VMMRZCallRing3Disable(pVCpu);
3104	Log4Func(("rcExit=%d LocalFF=%#RX64 GlobalFF=%#RX32\n", VBOXSTRICTRC_VAL(rcExit), (uint64_t)pVCpu->fLocalForcedActions,
3105	pVCpu->CTX_SUFF(pVM)->fGlobalForcedActions));
3106
3107	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
3108	if (pVCpu->hm.s.Event.fPending)
3109	{
3110	hmR0SvmPendingEventToTrpmTrap(pVCpu);
3111	Assert(!pVCpu->hm.s.Event.fPending);
3112	}
3113
3114	/* Sync. the necessary state for going back to ring-3. */
3115	hmR0SvmLeaveSession(pVCpu);
3116	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
3117
3118	/* Thread-context hooks are unregistered at this point!!! */
3119	/* Ring-3 callback notifications are unregistered at this point!!! */
3120
3121	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
3122	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
3123	\| CPUM_CHANGED_LDTR
3124	\| CPUM_CHANGED_GDTR
3125	\| CPUM_CHANGED_IDTR
3126	\| CPUM_CHANGED_TR
3127	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
3128	if ( pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging
3129	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
3130	{
3131	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
3132	}
3133
3134	/* Update the exit-to-ring 3 reason. */
3135	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
3136
3137	/* On our way back from ring-3, reload the guest-CPU state if it may change while in ring-3. */
3138	if ( rcExit != VINF_EM_RAW_INTERRUPT
3139	\|\| CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
3140	{
3141	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
3142	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
3143	}
3144
3145	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
3146	VMMRZCallRing3Enable(pVCpu);
3147
3148	/*
3149	* If we're emulating an instruction, we shouldn't have any TRPM traps pending
3150	* and if we're injecting an event we should have a TRPM trap pending.
3151	*/
3152	AssertReturnStmt(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu),
3153	pVCpu->hm.s.u32HMError = VBOXSTRICTRC_VAL(rcExit),
3154	VERR_SVM_IPE_5);
3155	AssertReturnStmt(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu),
3156	pVCpu->hm.s.u32HMError = VBOXSTRICTRC_VAL(rcExit),
3157	VERR_SVM_IPE_4);
3158
3159	return rcExit;
3160	}
3161
3162
3163	/**
3164	* Updates the use of TSC offsetting mode for the CPU and adjusts the necessary
3165	* intercepts.
3166	*
3167	* @param pVCpu The cross context virtual CPU structure.
3168	* @param pVmcb Pointer to the VM control block.
3169	*
3170	* @remarks No-long-jump zone!!!
3171	*/
3172	static void hmR0SvmUpdateTscOffsetting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3173	{
3174	/*
3175	* Avoid intercepting RDTSC/RDTSCP if we determined the host TSC (++) is stable
3176	* and in case of a nested-guest, if the nested-VMCB specifies it is not intercepting
3177	* RDTSC/RDTSCP as well.
3178	*/
3179	bool fParavirtTsc;
3180	uint64_t uTscOffset;
3181	bool const fCanUseRealTsc = TMCpuTickCanUseRealTSC(pVCpu->CTX_SUFF(pVM), pVCpu, &uTscOffset, &fParavirtTsc);
3182
3183	bool fIntercept;
3184	if (fCanUseRealTsc)
3185	fIntercept = hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP);
3186	else
3187	{
3188	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP);
3189	fIntercept = true;
3190	}
3191
3192	if (!fIntercept)
3193	{
3194	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3195	/* Apply the nested-guest VMCB's TSC offset over the guest TSC offset. */
3196	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
3197	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
3198	#endif
3199
3200	/* Update the TSC offset in the VMCB and the relevant clean bits. */
3201	pVmcb->ctrl.u64TSCOffset = uTscOffset;
3202	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
3203	}
3204
3205	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
3206	information before every VM-entry, hence we have nothing to do here at the moment. */
3207	if (fParavirtTsc)
3208	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
3209	}
3210
3211
3212	/**
3213	* Sets an event as a pending event to be injected into the guest.
3214	*
3215	* @param pVCpu The cross context virtual CPU structure.
3216	* @param pEvent Pointer to the SVM event.
3217	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
3218	* page-fault.
3219	*
3220	* @remarks Statistics counter assumes this is a guest event being reflected to
3221	* the guest i.e. 'StatInjectPendingReflect' is incremented always.
3222	*/
3223	DECLINLINE(void) hmR0SvmSetPendingEvent(PVMCPUCC pVCpu, PSVMEVENT pEvent, RTGCUINTPTR GCPtrFaultAddress)
3224	{
3225	Assert(!pVCpu->hm.s.Event.fPending);
3226	Assert(pEvent->n.u1Valid);
3227
3228	pVCpu->hm.s.Event.u64IntInfo = pEvent->u;
3229	pVCpu->hm.s.Event.fPending = true;
3230	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
3231
3232	Log4Func(("u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u, pEvent->n.u8Vector,
3233	(uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3234	}
3235
3236
3237	/**
3238	* Sets an divide error (\#DE) exception as pending-for-injection into the VM.
3239	*
3240	* @param pVCpu The cross context virtual CPU structure.
3241	*/
3242	DECLINLINE(void) hmR0SvmSetPendingXcptDE(PVMCPUCC pVCpu)
3243	{
3244	SVMEVENT Event;
3245	Event.u = 0;
3246	Event.n.u1Valid = 1;
3247	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3248	Event.n.u8Vector = X86_XCPT_DE;
3249	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3250	}
3251
3252
3253	/**
3254	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
3255	*
3256	* @param pVCpu The cross context virtual CPU structure.
3257	*/
3258	DECLINLINE(void) hmR0SvmSetPendingXcptUD(PVMCPUCC pVCpu)
3259	{
3260	SVMEVENT Event;
3261	Event.u = 0;
3262	Event.n.u1Valid = 1;
3263	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3264	Event.n.u8Vector = X86_XCPT_UD;
3265	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3266	}
3267
3268
3269	/**
3270	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
3271	*
3272	* @param pVCpu The cross context virtual CPU structure.
3273	*/
3274	DECLINLINE(void) hmR0SvmSetPendingXcptDB(PVMCPUCC pVCpu)
3275	{
3276	SVMEVENT Event;
3277	Event.u = 0;
3278	Event.n.u1Valid = 1;
3279	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3280	Event.n.u8Vector = X86_XCPT_DB;
3281	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3282	}
3283
3284
3285	/**
3286	* Sets a page fault (\#PF) exception as pending-for-injection into the VM.
3287	*
3288	* @param pVCpu The cross context virtual CPU structure.
3289	* @param u32ErrCode The error-code for the page-fault.
3290	* @param uFaultAddress The page fault address (CR2).
3291	*
3292	* @remarks This updates the guest CR2 with @a uFaultAddress!
3293	*/
3294	DECLINLINE(void) hmR0SvmSetPendingXcptPF(PVMCPUCC pVCpu, uint32_t u32ErrCode, RTGCUINTPTR uFaultAddress)
3295	{
3296	SVMEVENT Event;
3297	Event.u = 0;
3298	Event.n.u1Valid = 1;
3299	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3300	Event.n.u8Vector = X86_XCPT_PF;
3301	Event.n.u1ErrorCodeValid = 1;
3302	Event.n.u32ErrorCode = u32ErrCode;
3303
3304	/* Update CR2 of the guest. */
3305	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR2);
3306	if (pVCpu->cpum.GstCtx.cr2 != uFaultAddress)
3307	{
3308	pVCpu->cpum.GstCtx.cr2 = uFaultAddress;
3309	/* The VMCB clean bit for CR2 will be updated while re-loading the guest state. */
3310	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR2);
3311	}
3312
3313	hmR0SvmSetPendingEvent(pVCpu, &Event, uFaultAddress);
3314	}
3315
3316
3317	/**
3318	* Sets a math-fault (\#MF) exception as pending-for-injection into the VM.
3319	*
3320	* @param pVCpu The cross context virtual CPU structure.
3321	*/
3322	DECLINLINE(void) hmR0SvmSetPendingXcptMF(PVMCPUCC pVCpu)
3323	{
3324	SVMEVENT Event;
3325	Event.u = 0;
3326	Event.n.u1Valid = 1;
3327	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3328	Event.n.u8Vector = X86_XCPT_MF;
3329	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3330	}
3331
3332
3333	/**
3334	* Sets a double fault (\#DF) exception as pending-for-injection into the VM.
3335	*
3336	* @param pVCpu The cross context virtual CPU structure.
3337	*/
3338	DECLINLINE(void) hmR0SvmSetPendingXcptDF(PVMCPUCC pVCpu)
3339	{
3340	SVMEVENT Event;
3341	Event.u = 0;
3342	Event.n.u1Valid = 1;
3343	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3344	Event.n.u8Vector = X86_XCPT_DF;
3345	Event.n.u1ErrorCodeValid = 1;
3346	Event.n.u32ErrorCode = 0;
3347	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3348	}
3349
3350
3351	/**
3352	* Injects an event into the guest upon VMRUN by updating the relevant field
3353	* in the VMCB.
3354	*
3355	* @param pVCpu The cross context virtual CPU structure.
3356	* @param pVmcb Pointer to the guest VM control block.
3357	* @param pEvent Pointer to the event.
3358	*
3359	* @remarks No-long-jump zone!!!
3360	* @remarks Requires CR0!
3361	*/
3362	DECLINLINE(void) hmR0SvmInjectEventVmcb(PVMCPUCC pVCpu, PSVMVMCB pVmcb, PSVMEVENT pEvent)
3363	{
3364	Assert(!pVmcb->ctrl.EventInject.n.u1Valid);
3365	pVmcb->ctrl.EventInject.u = pEvent->u;
3366	if ( pVmcb->ctrl.EventInject.n.u3Type == SVM_EVENT_EXCEPTION
3367	\|\| pVmcb->ctrl.EventInject.n.u3Type == SVM_EVENT_NMI)
3368	{
3369	Assert(pEvent->n.u8Vector <= X86_XCPT_LAST);
3370	STAM_COUNTER_INC(&pVCpu->hm.s.aStatInjectedXcpts[pEvent->n.u8Vector]);
3371	}
3372	else
3373	STAM_COUNTER_INC(&pVCpu->hm.s.aStatInjectedIrqs[pEvent->n.u8Vector & MASK_INJECT_IRQ_STAT]);
3374	RT_NOREF(pVCpu);
3375
3376	Log4Func(("u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u, pEvent->n.u8Vector,
3377	(uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3378	}
3379
3380
3381
3382	/**
3383	* Converts any TRPM trap into a pending HM event. This is typically used when
3384	* entering from ring-3 (not longjmp returns).
3385	*
3386	* @param pVCpu The cross context virtual CPU structure.
3387	*/
3388	static void hmR0SvmTrpmTrapToPendingEvent(PVMCPUCC pVCpu)
3389	{
3390	Assert(TRPMHasTrap(pVCpu));
3391	Assert(!pVCpu->hm.s.Event.fPending);
3392
3393	uint8_t uVector;
3394	TRPMEVENT enmTrpmEvent;
3395	uint32_t uErrCode;
3396	RTGCUINTPTR GCPtrFaultAddress;
3397	uint8_t cbInstr;
3398
3399	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr, NULL /* pfIcebp */);
3400	AssertRC(rc);
3401
3402	SVMEVENT Event;
3403	Event.u = 0;
3404	Event.n.u1Valid = 1;
3405	Event.n.u8Vector = uVector;
3406
3407	/* Refer AMD spec. 15.20 "Event Injection" for the format. */
3408	if (enmTrpmEvent == TRPM_TRAP)
3409	{
3410	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3411	switch (uVector)
3412	{
3413	case X86_XCPT_NMI:
3414	{
3415	Event.n.u3Type = SVM_EVENT_NMI;
3416	break;
3417	}
3418
3419	case X86_XCPT_BP:
3420	case X86_XCPT_OF:
3421	AssertMsgFailed(("Invalid TRPM vector %d for event type %d\n", uVector, enmTrpmEvent));
3422	RT_FALL_THRU();
3423
3424	case X86_XCPT_PF:
3425	case X86_XCPT_DF:
3426	case X86_XCPT_TS:
3427	case X86_XCPT_NP:
3428	case X86_XCPT_SS:
3429	case X86_XCPT_GP:
3430	case X86_XCPT_AC:
3431	{
3432	Event.n.u1ErrorCodeValid = 1;
3433	Event.n.u32ErrorCode = uErrCode;
3434	break;
3435	}
3436	}
3437	}
3438	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
3439	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3440	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
3441	Event.n.u3Type = SVM_EVENT_SOFTWARE_INT;
3442	else
3443	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
3444
3445	rc = TRPMResetTrap(pVCpu);
3446	AssertRC(rc);
3447
3448	Log4(("TRPM->HM event: u=%#RX64 u8Vector=%#x uErrorCodeValid=%RTbool uErrorCode=%#RX32\n", Event.u, Event.n.u8Vector,
3449	!!Event.n.u1ErrorCodeValid, Event.n.u32ErrorCode));
3450
3451	hmR0SvmSetPendingEvent(pVCpu, &Event, GCPtrFaultAddress);
3452	}
3453
3454
3455	/**
3456	* Converts any pending SVM event into a TRPM trap. Typically used when leaving
3457	* AMD-V to execute any instruction.
3458	*
3459	* @param pVCpu The cross context virtual CPU structure.
3460	*/
3461	static void hmR0SvmPendingEventToTrpmTrap(PVMCPUCC pVCpu)
3462	{
3463	Assert(pVCpu->hm.s.Event.fPending);
3464	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
3465
3466	SVMEVENT Event;
3467	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3468
3469	uint8_t uVector = Event.n.u8Vector;
3470	TRPMEVENT enmTrapType = HMSvmEventToTrpmEventType(&Event, uVector);
3471
3472	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, Event.n.u3Type));
3473
3474	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
3475	AssertRC(rc);
3476
3477	if (Event.n.u1ErrorCodeValid)
3478	TRPMSetErrorCode(pVCpu, Event.n.u32ErrorCode);
3479
3480	if ( enmTrapType == TRPM_TRAP
3481	&& uVector == X86_XCPT_PF)
3482	{
3483	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
3484	Assert(pVCpu->hm.s.Event.GCPtrFaultAddress == CPUMGetGuestCR2(pVCpu));
3485	}
3486	else if (enmTrapType == TRPM_SOFTWARE_INT)
3487	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
3488	pVCpu->hm.s.Event.fPending = false;
3489	}
3490
3491
3492	/**
3493	* Checks if the guest (or nested-guest) has an interrupt shadow active right
3494	* now.
3495	*
3496	* @returns @c true if the interrupt shadow is active, @c false otherwise.
3497	* @param pVCpu The cross context virtual CPU structure.
3498	*
3499	* @remarks No-long-jump zone!!!
3500	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3501	*/
3502	static bool hmR0SvmIsIntrShadowActive(PVMCPUCC pVCpu)
3503	{
3504	/*
3505	* Instructions like STI and MOV SS inhibit interrupts till the next instruction
3506	* completes. Check if we should inhibit interrupts or clear any existing
3507	* interrupt inhibition.
3508	*/
3509	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3510	{
3511	if (pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
3512	{
3513	/*
3514	* We can clear the inhibit force flag as even if we go back to the recompiler
3515	* without executing guest code in AMD-V, the flag's condition to be cleared is
3516	* met and thus the cleared state is correct.
3517	*/
3518	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3519	return false;
3520	}
3521	return true;
3522	}
3523	return false;
3524	}
3525
3526
3527	/**
3528	* Sets the virtual interrupt intercept control in the VMCB.
3529	*
3530	* @param pVCpu The cross context virtual CPU structure.
3531	* @param pVmcb Pointer to the VM control block.
3532	*/
3533	static void hmR0SvmSetIntWindowExiting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3534	{
3535	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx); NOREF(pVCpu);
3536
3537	/*
3538	* When AVIC isn't supported, set up an interrupt window to cause a #VMEXIT when the guest
3539	* is ready to accept interrupts. At #VMEXIT, we then get the interrupt from the APIC
3540	* (updating ISR at the right time) and inject the interrupt.
3541	*
3542	* With AVIC is supported, we could make use of the asynchronously delivery without
3543	* #VMEXIT and we would be passing the AVIC page to SVM.
3544	*
3545	* In AMD-V, an interrupt window is achieved using a combination of V_IRQ (an interrupt
3546	* is pending), V_IGN_TPR (ignore TPR priorities) and the VINTR intercept all being set.
3547	*/
3548	Assert(pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR);
3549	pVmcb->ctrl.IntCtrl.n.u1VIrqPending = 1;
3550	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
3551	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_VINTR);
3552	Log4(("Set VINTR intercept\n"));
3553	}
3554
3555
3556	/**
3557	* Clears the virtual interrupt intercept control in the VMCB as
3558	* we are figured the guest is unable process any interrupts
3559	* at this point of time.
3560	*
3561	* @param pVCpu The cross context virtual CPU structure.
3562	* @param pVmcb Pointer to the VM control block.
3563	*/
3564	static void hmR0SvmClearIntWindowExiting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3565	{
3566	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx); NOREF(pVCpu);
3567
3568	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
3569	if ( pVmcbCtrl->IntCtrl.n.u1VIrqPending
3570	\|\| (pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_VINTR))
3571	{
3572	pVmcbCtrl->IntCtrl.n.u1VIrqPending = 0;
3573	pVmcbCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
3574	hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_VINTR);
3575	Log4(("Cleared VINTR intercept\n"));
3576	}
3577	}
3578
3579
3580	/**
3581	* Evaluates the event to be delivered to the guest and sets it as the pending
3582	* event.
3583	*
3584	* @returns Strict VBox status code.
3585	* @param pVCpu The cross context virtual CPU structure.
3586	* @param pSvmTransient Pointer to the SVM transient structure.
3587	*/
3588	static VBOXSTRICTRC hmR0SvmEvaluatePendingEvent(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient)
3589	{
3590	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3591	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_HWVIRT
3592	\| CPUMCTX_EXTRN_RFLAGS
3593	\| CPUMCTX_EXTRN_INHIBIT_INT
3594	\| CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ);
3595
3596	Assert(!pVCpu->hm.s.Event.fPending);
3597	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
3598	Assert(pVmcb);
3599
3600	bool const fGif = CPUMGetGuestGif(pCtx);
3601	bool const fIntShadow = hmR0SvmIsIntrShadowActive(pVCpu);
3602	bool const fBlockNmi = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
3603
3604	Log4Func(("fGif=%RTbool fBlockNmi=%RTbool fIntShadow=%RTbool fIntPending=%RTbool fNmiPending=%RTbool\n",
3605	fGif, fBlockNmi, fIntShadow, VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC),
3606	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)));
3607
3608	/** @todo SMI. SMIs take priority over NMIs. */
3609
3610	/*
3611	* Check if the guest or nested-guest can receive NMIs.
3612	* Nested NMIs are not allowed, see AMD spec. 8.1.4 "Masking External Interrupts".
3613	* NMIs take priority over maskable interrupts, see AMD spec. 8.5 "Priorities".
3614	*/
3615	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)
3616	&& !fBlockNmi)
3617	{
3618	if ( fGif
3619	&& !fIntShadow)
3620	{
3621	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3622	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_NMI))
3623	{
3624	Log4(("Intercepting NMI -> #VMEXIT\n"));
3625	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3626	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_NMI, 0, 0);
3627	}
3628	#endif
3629	Log4(("Setting NMI pending for injection\n"));
3630	SVMEVENT Event;
3631	Event.u = 0;
3632	Event.n.u1Valid = 1;
3633	Event.n.u8Vector = X86_XCPT_NMI;
3634	Event.n.u3Type = SVM_EVENT_NMI;
3635	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3636	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
3637	}
3638	else if (!fGif)
3639	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3640	else if (!pSvmTransient->fIsNestedGuest)
3641	hmR0SvmSetIntWindowExiting(pVCpu, pVmcb);
3642	/* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCB controls. */
3643	}
3644	/*
3645	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt()
3646	* returns a valid interrupt we -must- deliver the interrupt. We can no longer re-request
3647	* it from the APIC device.
3648	*
3649	* For nested-guests, physical interrupts always take priority over virtual interrupts.
3650	* We don't need to inject nested-guest virtual interrupts here, we can let the hardware
3651	* do that work when we execute nested-guest code esp. since all the required information
3652	* is in the VMCB, unlike physical interrupts where we need to fetch the interrupt from
3653	* the virtual interrupt controller.
3654	*
3655	* See AMD spec. 15.21.4 "Injecting Virtual (INTR) Interrupts".
3656	*/
3657	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
3658	&& !pVCpu->hm.s.fSingleInstruction)
3659	{
3660	bool const fBlockInt = !pSvmTransient->fIsNestedGuest ? !(pCtx->eflags.u32 & X86_EFL_IF)
3661	: CPUMIsGuestSvmPhysIntrEnabled(pVCpu, pCtx);
3662	if ( fGif
3663	&& !fBlockInt
3664	&& !fIntShadow)
3665	{
3666	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3667	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INTR))
3668	{
3669	Log4(("Intercepting INTR -> #VMEXIT\n"));
3670	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3671	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_INTR, 0, 0);
3672	}
3673	#endif
3674	uint8_t u8Interrupt;
3675	int rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
3676	if (RT_SUCCESS(rc))
3677	{
3678	Log4(("Setting external interrupt %#x pending for injection\n", u8Interrupt));
3679	SVMEVENT Event;
3680	Event.u = 0;
3681	Event.n.u1Valid = 1;
3682	Event.n.u8Vector = u8Interrupt;
3683	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3684	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3685	}
3686	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
3687	{
3688	/*
3689	* AMD-V has no TPR thresholding feature. TPR and the force-flag will be
3690	* updated eventually when the TPR is written by the guest.
3691	*/
3692	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
3693	}
3694	else
3695	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
3696	}
3697	else if (!fGif)
3698	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3699	else if (!pSvmTransient->fIsNestedGuest)
3700	hmR0SvmSetIntWindowExiting(pVCpu, pVmcb);
3701	/* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCB controls. */
3702	}
3703
3704	return VINF_SUCCESS;
3705	}
3706
3707
3708	/**
3709	* Injects any pending events into the guest (or nested-guest).
3710	*
3711	* @param pVCpu The cross context virtual CPU structure.
3712	* @param pVmcb Pointer to the VM control block.
3713	*
3714	* @remarks Must only be called when we are guaranteed to enter
3715	* hardware-assisted SVM execution and not return to ring-3
3716	* prematurely.
3717	*/
3718	static void hmR0SvmInjectPendingEvent(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3719	{
3720	Assert(!TRPMHasTrap(pVCpu));
3721	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3722
3723	bool const fIntShadow = hmR0SvmIsIntrShadowActive(pVCpu);
3724	#ifdef VBOX_STRICT
3725	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3726	bool const fGif = CPUMGetGuestGif(pCtx);
3727	bool fAllowInt = fGif;
3728	if (fGif)
3729	{
3730	/*
3731	* For nested-guests we have no way to determine if we're injecting a physical or
3732	* virtual interrupt at this point. Hence the partial verification below.
3733	*/
3734	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
3735	fAllowInt = CPUMIsGuestSvmPhysIntrEnabled(pVCpu, pCtx) \|\| CPUMIsGuestSvmVirtIntrEnabled(pVCpu, pCtx);
3736	else
3737	fAllowInt = RT_BOOL(pCtx->eflags.u32 & X86_EFL_IF);
3738	}
3739	#endif
3740
3741	if (pVCpu->hm.s.Event.fPending)
3742	{
3743	SVMEVENT Event;
3744	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3745	Assert(Event.n.u1Valid);
3746
3747	/*
3748	* Validate event injection pre-conditions.
3749	*/
3750	if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3751	{
3752	Assert(fAllowInt);
3753	Assert(!fIntShadow);
3754	}
3755	else if (Event.n.u3Type == SVM_EVENT_NMI)
3756	{
3757	Assert(fGif);
3758	Assert(!fIntShadow);
3759	}
3760
3761	/*
3762	* Before injecting an NMI we must set VMCPU_FF_BLOCK_NMIS to prevent nested NMIs. We
3763	* do this only when we are surely going to inject the NMI as otherwise if we return
3764	* to ring-3 prematurely we could leave NMIs blocked indefinitely upon re-entry into
3765	* SVM R0.
3766	*
3767	* With VT-x, this is handled by the Guest interruptibility information VMCS field
3768	* which will set the VMCS field after actually delivering the NMI which we read on
3769	* VM-exit to determine the state.
3770	*/
3771	if ( Event.n.u3Type == SVM_EVENT_NMI
3772	&& Event.n.u8Vector == X86_XCPT_NMI
3773	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
3774	{
3775	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
3776	}
3777
3778	/*
3779	* Inject it (update VMCB for injection by the hardware).
3780	*/
3781	Log4(("Injecting pending HM event\n"));
3782	hmR0SvmInjectEventVmcb(pVCpu, pVmcb, &Event);
3783	pVCpu->hm.s.Event.fPending = false;
3784
3785	if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3786	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
3787	else
3788	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
3789	}
3790	else
3791	Assert(pVmcb->ctrl.EventInject.n.u1Valid == 0);
3792
3793	/*
3794	* We could have injected an NMI through IEM and continue guest execution using
3795	* hardware-assisted SVM. In which case, we would not have any events pending (above)
3796	* but we still need to intercept IRET in order to eventually clear NMI inhibition.
3797	*/
3798	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
3799	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_IRET);
3800
3801	/*
3802	* Update the guest interrupt shadow in the guest (or nested-guest) VMCB.
3803	*
3804	* For nested-guests: We need to update it too for the scenario where IEM executes
3805	* the nested-guest but execution later continues here with an interrupt shadow active.
3806	*/
3807	pVmcb->ctrl.IntShadow.n.u1IntShadow = fIntShadow;
3808	}
3809
3810
3811	/**
3812	* Reports world-switch error and dumps some useful debug info.
3813	*
3814	* @param pVCpu The cross context virtual CPU structure.
3815	* @param rcVMRun The return code from VMRUN (or
3816	* VERR_SVM_INVALID_GUEST_STATE for invalid
3817	* guest-state).
3818	*/
3819	static void hmR0SvmReportWorldSwitchError(PVMCPUCC pVCpu, int rcVMRun)
3820	{
3821	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
3822	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
3823	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3824
3825	if (rcVMRun == VERR_SVM_INVALID_GUEST_STATE)
3826	{
3827	#ifdef VBOX_STRICT
3828	hmR0DumpRegs(pVCpu, HM_DUMP_REG_FLAGS_ALL);
3829	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
3830	Log4(("ctrl.u32VmcbCleanBits %#RX32\n", pVmcb->ctrl.u32VmcbCleanBits));
3831	Log4(("ctrl.u16InterceptRdCRx %#x\n", pVmcb->ctrl.u16InterceptRdCRx));
3832	Log4(("ctrl.u16InterceptWrCRx %#x\n", pVmcb->ctrl.u16InterceptWrCRx));
3833	Log4(("ctrl.u16InterceptRdDRx %#x\n", pVmcb->ctrl.u16InterceptRdDRx));
3834	Log4(("ctrl.u16InterceptWrDRx %#x\n", pVmcb->ctrl.u16InterceptWrDRx));
3835	Log4(("ctrl.u32InterceptXcpt %#x\n", pVmcb->ctrl.u32InterceptXcpt));
3836	Log4(("ctrl.u64InterceptCtrl %#RX64\n", pVmcb->ctrl.u64InterceptCtrl));
3837	Log4(("ctrl.u64IOPMPhysAddr %#RX64\n", pVmcb->ctrl.u64IOPMPhysAddr));
3838	Log4(("ctrl.u64MSRPMPhysAddr %#RX64\n", pVmcb->ctrl.u64MSRPMPhysAddr));
3839	Log4(("ctrl.u64TSCOffset %#RX64\n", pVmcb->ctrl.u64TSCOffset));
3840
3841	Log4(("ctrl.TLBCtrl.u32ASID %#x\n", pVmcb->ctrl.TLBCtrl.n.u32ASID));
3842	Log4(("ctrl.TLBCtrl.u8TLBFlush %#x\n", pVmcb->ctrl.TLBCtrl.n.u8TLBFlush));
3843	Log4(("ctrl.TLBCtrl.u24Reserved %#x\n", pVmcb->ctrl.TLBCtrl.n.u24Reserved));
3844
3845	Log4(("ctrl.IntCtrl.u8VTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u8VTPR));
3846	Log4(("ctrl.IntCtrl.u1VIrqPending %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIrqPending));
3847	Log4(("ctrl.IntCtrl.u1VGif %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGif));
3848	Log4(("ctrl.IntCtrl.u6Reserved0 %#x\n", pVmcb->ctrl.IntCtrl.n.u6Reserved));
3849	Log4(("ctrl.IntCtrl.u4VIntrPrio %#x\n", pVmcb->ctrl.IntCtrl.n.u4VIntrPrio));
3850	Log4(("ctrl.IntCtrl.u1IgnoreTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR));
3851	Log4(("ctrl.IntCtrl.u3Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u3Reserved));
3852	Log4(("ctrl.IntCtrl.u1VIntrMasking %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIntrMasking));
3853	Log4(("ctrl.IntCtrl.u1VGifEnable %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGifEnable));
3854	Log4(("ctrl.IntCtrl.u5Reserved1 %#x\n", pVmcb->ctrl.IntCtrl.n.u5Reserved));
3855	Log4(("ctrl.IntCtrl.u8VIntrVector %#x\n", pVmcb->ctrl.IntCtrl.n.u8VIntrVector));
3856	Log4(("ctrl.IntCtrl.u24Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u24Reserved));
3857
3858	Log4(("ctrl.IntShadow.u1IntShadow %#x\n", pVmcb->ctrl.IntShadow.n.u1IntShadow));
3859	Log4(("ctrl.IntShadow.u1GuestIntMask %#x\n", pVmcb->ctrl.IntShadow.n.u1GuestIntMask));
3860	Log4(("ctrl.u64ExitCode %#RX64\n", pVmcb->ctrl.u64ExitCode));
3861	Log4(("ctrl.u64ExitInfo1 %#RX64\n", pVmcb->ctrl.u64ExitInfo1));
3862	Log4(("ctrl.u64ExitInfo2 %#RX64\n", pVmcb->ctrl.u64ExitInfo2));
3863	Log4(("ctrl.ExitIntInfo.u8Vector %#x\n", pVmcb->ctrl.ExitIntInfo.n.u8Vector));
3864	Log4(("ctrl.ExitIntInfo.u3Type %#x\n", pVmcb->ctrl.ExitIntInfo.n.u3Type));
3865	Log4(("ctrl.ExitIntInfo.u1ErrorCodeValid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid));
3866	Log4(("ctrl.ExitIntInfo.u19Reserved %#x\n", pVmcb->ctrl.ExitIntInfo.n.u19Reserved));
3867	Log4(("ctrl.ExitIntInfo.u1Valid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1Valid));
3868	Log4(("ctrl.ExitIntInfo.u32ErrorCode %#x\n", pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode));
3869	Log4(("ctrl.NestedPagingCtrl.u1NestedPaging %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1NestedPaging));
3870	Log4(("ctrl.NestedPagingCtrl.u1Sev %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1Sev));
3871	Log4(("ctrl.NestedPagingCtrl.u1SevEs %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1SevEs));
3872	Log4(("ctrl.EventInject.u8Vector %#x\n", pVmcb->ctrl.EventInject.n.u8Vector));
3873	Log4(("ctrl.EventInject.u3Type %#x\n", pVmcb->ctrl.EventInject.n.u3Type));
3874	Log4(("ctrl.EventInject.u1ErrorCodeValid %#x\n", pVmcb->ctrl.EventInject.n.u1ErrorCodeValid));
3875	Log4(("ctrl.EventInject.u19Reserved %#x\n", pVmcb->ctrl.EventInject.n.u19Reserved));
3876	Log4(("ctrl.EventInject.u1Valid %#x\n", pVmcb->ctrl.EventInject.n.u1Valid));
3877	Log4(("ctrl.EventInject.u32ErrorCode %#x\n", pVmcb->ctrl.EventInject.n.u32ErrorCode));
3878
3879	Log4(("ctrl.u64NestedPagingCR3 %#RX64\n", pVmcb->ctrl.u64NestedPagingCR3));
3880
3881	Log4(("ctrl.LbrVirt.u1LbrVirt %#x\n", pVmcb->ctrl.LbrVirt.n.u1LbrVirt));
3882	Log4(("ctrl.LbrVirt.u1VirtVmsaveVmload %#x\n", pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload));
3883
3884	Log4(("guest.CS.u16Sel %RTsel\n", pVmcb->guest.CS.u16Sel));
3885	Log4(("guest.CS.u16Attr %#x\n", pVmcb->guest.CS.u16Attr));
3886	Log4(("guest.CS.u32Limit %#RX32\n", pVmcb->guest.CS.u32Limit));
3887	Log4(("guest.CS.u64Base %#RX64\n", pVmcb->guest.CS.u64Base));
3888	Log4(("guest.DS.u16Sel %#RTsel\n", pVmcb->guest.DS.u16Sel));
3889	Log4(("guest.DS.u16Attr %#x\n", pVmcb->guest.DS.u16Attr));
3890	Log4(("guest.DS.u32Limit %#RX32\n", pVmcb->guest.DS.u32Limit));
3891	Log4(("guest.DS.u64Base %#RX64\n", pVmcb->guest.DS.u64Base));
3892	Log4(("guest.ES.u16Sel %RTsel\n", pVmcb->guest.ES.u16Sel));
3893	Log4(("guest.ES.u16Attr %#x\n", pVmcb->guest.ES.u16Attr));
3894	Log4(("guest.ES.u32Limit %#RX32\n", pVmcb->guest.ES.u32Limit));
3895	Log4(("guest.ES.u64Base %#RX64\n", pVmcb->guest.ES.u64Base));
3896	Log4(("guest.FS.u16Sel %RTsel\n", pVmcb->guest.FS.u16Sel));
3897	Log4(("guest.FS.u16Attr %#x\n", pVmcb->guest.FS.u16Attr));
3898	Log4(("guest.FS.u32Limit %#RX32\n", pVmcb->guest.FS.u32Limit));
3899	Log4(("guest.FS.u64Base %#RX64\n", pVmcb->guest.FS.u64Base));
3900	Log4(("guest.GS.u16Sel %RTsel\n", pVmcb->guest.GS.u16Sel));
3901	Log4(("guest.GS.u16Attr %#x\n", pVmcb->guest.GS.u16Attr));
3902	Log4(("guest.GS.u32Limit %#RX32\n", pVmcb->guest.GS.u32Limit));
3903	Log4(("guest.GS.u64Base %#RX64\n", pVmcb->guest.GS.u64Base));
3904
3905	Log4(("guest.GDTR.u32Limit %#RX32\n", pVmcb->guest.GDTR.u32Limit));
3906	Log4(("guest.GDTR.u64Base %#RX64\n", pVmcb->guest.GDTR.u64Base));
3907
3908	Log4(("guest.LDTR.u16Sel %RTsel\n", pVmcb->guest.LDTR.u16Sel));
3909	Log4(("guest.LDTR.u16Attr %#x\n", pVmcb->guest.LDTR.u16Attr));
3910	Log4(("guest.LDTR.u32Limit %#RX32\n", pVmcb->guest.LDTR.u32Limit));
3911	Log4(("guest.LDTR.u64Base %#RX64\n", pVmcb->guest.LDTR.u64Base));
3912
3913	Log4(("guest.IDTR.u32Limit %#RX32\n", pVmcb->guest.IDTR.u32Limit));
3914	Log4(("guest.IDTR.u64Base %#RX64\n", pVmcb->guest.IDTR.u64Base));
3915
3916	Log4(("guest.TR.u16Sel %RTsel\n", pVmcb->guest.TR.u16Sel));
3917	Log4(("guest.TR.u16Attr %#x\n", pVmcb->guest.TR.u16Attr));
3918	Log4(("guest.TR.u32Limit %#RX32\n", pVmcb->guest.TR.u32Limit));
3919	Log4(("guest.TR.u64Base %#RX64\n", pVmcb->guest.TR.u64Base));
3920
3921	Log4(("guest.u8CPL %#x\n", pVmcb->guest.u8CPL));
3922	Log4(("guest.u64CR0 %#RX64\n", pVmcb->guest.u64CR0));
3923	Log4(("guest.u64CR2 %#RX64\n", pVmcb->guest.u64CR2));
3924	Log4(("guest.u64CR3 %#RX64\n", pVmcb->guest.u64CR3));
3925	Log4(("guest.u64CR4 %#RX64\n", pVmcb->guest.u64CR4));
3926	Log4(("guest.u64DR6 %#RX64\n", pVmcb->guest.u64DR6));
3927	Log4(("guest.u64DR7 %#RX64\n", pVmcb->guest.u64DR7));
3928
3929	Log4(("guest.u64RIP %#RX64\n", pVmcb->guest.u64RIP));
3930	Log4(("guest.u64RSP %#RX64\n", pVmcb->guest.u64RSP));
3931	Log4(("guest.u64RAX %#RX64\n", pVmcb->guest.u64RAX));
3932	Log4(("guest.u64RFlags %#RX64\n", pVmcb->guest.u64RFlags));
3933
3934	Log4(("guest.u64SysEnterCS %#RX64\n", pVmcb->guest.u64SysEnterCS));
3935	Log4(("guest.u64SysEnterEIP %#RX64\n", pVmcb->guest.u64SysEnterEIP));
3936	Log4(("guest.u64SysEnterESP %#RX64\n", pVmcb->guest.u64SysEnterESP));
3937
3938	Log4(("guest.u64EFER %#RX64\n", pVmcb->guest.u64EFER));
3939	Log4(("guest.u64STAR %#RX64\n", pVmcb->guest.u64STAR));
3940	Log4(("guest.u64LSTAR %#RX64\n", pVmcb->guest.u64LSTAR));
3941	Log4(("guest.u64CSTAR %#RX64\n", pVmcb->guest.u64CSTAR));
3942	Log4(("guest.u64SFMASK %#RX64\n", pVmcb->guest.u64SFMASK));
3943	Log4(("guest.u64KernelGSBase %#RX64\n", pVmcb->guest.u64KernelGSBase));
3944	Log4(("guest.u64PAT %#RX64\n", pVmcb->guest.u64PAT));
3945	Log4(("guest.u64DBGCTL %#RX64\n", pVmcb->guest.u64DBGCTL));
3946	Log4(("guest.u64BR_FROM %#RX64\n", pVmcb->guest.u64BR_FROM));
3947	Log4(("guest.u64BR_TO %#RX64\n", pVmcb->guest.u64BR_TO));
3948	Log4(("guest.u64LASTEXCPFROM %#RX64\n", pVmcb->guest.u64LASTEXCPFROM));
3949	Log4(("guest.u64LASTEXCPTO %#RX64\n", pVmcb->guest.u64LASTEXCPTO));
3950
3951	NOREF(pVmcb);
3952	#endif /* VBOX_STRICT */
3953	}
3954	else
3955	Log4Func(("rcVMRun=%d\n", rcVMRun));
3956	}
3957
3958
3959	/**
3960	* Check per-VM and per-VCPU force flag actions that require us to go back to
3961	* ring-3 for one reason or another.
3962	*
3963	* @returns Strict VBox status code (information status code included).
3964	* @retval VINF_SUCCESS if we don't have any actions that require going back to
3965	* ring-3.
3966	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
3967	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
3968	* interrupts)
3969	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
3970	* all EMTs to be in ring-3.
3971	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
3972	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
3973	* to the EM loop.
3974	*
3975	* @param pVCpu The cross context virtual CPU structure.
3976	*/
3977	static VBOXSTRICTRC hmR0SvmCheckForceFlags(PVMCPUCC pVCpu)
3978	{
3979	Assert(VMMRZCallRing3IsEnabled(pVCpu));
3980
3981	/* Could happen as a result of longjump. */
3982	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
3983	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
3984
3985	/* Update pending interrupts into the APIC's IRR. */
3986	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
3987	APICUpdatePendingInterrupts(pVCpu);
3988
3989	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3990	if ( VM_FF_IS_ANY_SET(pVM, !pVCpu->hm.s.fSingleInstruction
3991	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
3992	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, !pVCpu->hm.s.fSingleInstruction
3993	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
3994	{
3995	/* Pending PGM C3 sync. */
3996	if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
3997	{
3998	int rc = PGMSyncCR3(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.cr4,
3999	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
4000	if (rc != VINF_SUCCESS)
4001	{
4002	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
4003	return rc;
4004	}
4005	}
4006
4007	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
4008	/* -XXX- what was that about single stepping? */
4009	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
4010	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
4011	{
4012	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
4013	int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
4014	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
4015	return rc;
4016	}
4017
4018	/* Pending VM request packets, such as hardware interrupts. */
4019	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
4020	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
4021	{
4022	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchVmReq);
4023	Log4Func(("Pending VM request forcing us back to ring-3\n"));
4024	return VINF_EM_PENDING_REQUEST;
4025	}
4026
4027	/* Pending PGM pool flushes. */
4028	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
4029	{
4030	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPgmPoolFlush);
4031	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
4032	return VINF_PGM_POOL_FLUSH_PENDING;
4033	}
4034
4035	/* Pending DMA requests. */
4036	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
4037	{
4038	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchDma);
4039	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
4040	return VINF_EM_RAW_TO_R3;
4041	}
4042	}
4043
4044	return VINF_SUCCESS;
4045	}
4046
4047
4048	/**
4049	* Does the preparations before executing guest code in AMD-V.
4050	*
4051	* This may cause longjmps to ring-3 and may even result in rescheduling to the
4052	* recompiler. We must be cautious what we do here regarding committing
4053	* guest-state information into the VMCB assuming we assuredly execute the guest
4054	* in AMD-V. If we fall back to the recompiler after updating the VMCB and
4055	* clearing the common-state (TRPM/forceflags), we must undo those changes so
4056	* that the recompiler can (and should) use them when it resumes guest
4057	* execution. Otherwise such operations must be done when we can no longer
4058	* exit to ring-3.
4059	*
4060	* @returns Strict VBox status code (informational status codes included).
4061	* @retval VINF_SUCCESS if we can proceed with running the guest.
4062	* @retval VINF_* scheduling changes, we have to go back to ring-3.
4063	*
4064	* @param pVCpu The cross context virtual CPU structure.
4065	* @param pSvmTransient Pointer to the SVM transient structure.
4066	*/
4067	static VBOXSTRICTRC hmR0SvmPreRunGuest(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4068	{
4069	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
4070
4071	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
4072	if (pSvmTransient->fIsNestedGuest)
4073	{
4074	Log2(("hmR0SvmPreRunGuest: Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
4075	return VINF_EM_RESCHEDULE_REM;
4076	}
4077	#endif
4078
4079	/* Check force flag actions that might require us to go back to ring-3. */
4080	VBOXSTRICTRC rc = hmR0SvmCheckForceFlags(pVCpu);
4081	if (rc != VINF_SUCCESS)
4082	return rc;
4083
4084	if (TRPMHasTrap(pVCpu))
4085	hmR0SvmTrpmTrapToPendingEvent(pVCpu);
4086	else if (!pVCpu->hm.s.Event.fPending)
4087	{
4088	rc = hmR0SvmEvaluatePendingEvent(pVCpu, pSvmTransient);
4089	if ( rc != VINF_SUCCESS
4090	\|\| pSvmTransient->fIsNestedGuest != CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
4091	{
4092	/* If a nested-guest VM-exit occurred, bail. */
4093	if (pSvmTransient->fIsNestedGuest)
4094	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
4095	return rc;
4096	}
4097	}
4098
4099	/*
4100	* On the oldest AMD-V systems, we may not get enough information to reinject an NMI.
4101	* Just do it in software, see @bugref{8411}.
4102	* NB: If we could continue a task switch exit we wouldn't need to do this.
4103	*/
4104	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4105	if (RT_UNLIKELY( !g_fHmSvmFeatures
4106	&& pVCpu->hm.s.Event.fPending
4107	&& SVM_EVENT_GET_TYPE(pVCpu->hm.s.Event.u64IntInfo) == SVM_EVENT_NMI))
4108	return VINF_EM_RAW_INJECT_TRPM_EVENT;
4109
4110	#ifdef HMSVM_SYNC_FULL_GUEST_STATE
4111	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
4112	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
4113	#endif
4114
4115	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4116	/*
4117	* Set up the nested-guest VMCB for execution using hardware-assisted SVM.
4118	*/
4119	if (pSvmTransient->fIsNestedGuest)
4120	hmR0SvmSetupVmcbNested(pVCpu);
4121	#endif
4122
4123	/*
4124	* Export the guest state bits that are not shared with the host in any way as we can
4125	* longjmp or get preempted in the midst of exporting some of the state.
4126	*/
4127	rc = hmR0SvmExportGuestState(pVCpu, pSvmTransient);
4128	AssertRCReturn(rc, rc);
4129	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
4130
4131	/* Ensure we've cached (and hopefully modified) the nested-guest VMCB for execution using hardware-assisted SVM. */
4132	Assert(!pSvmTransient->fIsNestedGuest \|\| pVCpu->hm.s.svm.NstGstVmcbCache.fCacheValid);
4133
4134	/*
4135	* If we're not intercepting TPR changes in the guest, save the guest TPR before the
4136	* world-switch so we can update it on the way back if the guest changed the TPR.
4137	*/
4138	if (pVCpu->hmr0.s.svm.fSyncVTpr)
4139	{
4140	Assert(!pSvmTransient->fIsNestedGuest);
4141	PCSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
4142	if (pVM->hm.s.fTprPatchingActive)
4143	pSvmTransient->u8GuestTpr = pVmcb->guest.u64LSTAR;
4144	else
4145	pSvmTransient->u8GuestTpr = pVmcb->ctrl.IntCtrl.n.u8VTPR;
4146	}
4147
4148	/*
4149	* No longjmps to ring-3 from this point on!!!
4150	*
4151	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4152	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4153	*/
4154	VMMRZCallRing3Disable(pVCpu);
4155
4156	/*
4157	* We disable interrupts so that we don't miss any interrupts that would flag preemption
4158	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
4159	* preemption disabled for a while. Since this is purly to aid the
4160	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
4161	* disable interrupt on NT.
4162	*
4163	* We need to check for force-flags that could've possible been altered since we last
4164	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
4165	* see @bugref{6398}).
4166	*
4167	* We also check a couple of other force-flags as a last opportunity to get the EMT back
4168	* to ring-3 before executing guest code.
4169	*/
4170	pSvmTransient->fEFlags = ASMIntDisableFlags();
4171	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
4172	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
4173	{
4174	ASMSetFlags(pSvmTransient->fEFlags);
4175	VMMRZCallRing3Enable(pVCpu);
4176	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
4177	return VINF_EM_RAW_TO_R3;
4178	}
4179	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
4180	{
4181	ASMSetFlags(pSvmTransient->fEFlags);
4182	VMMRZCallRing3Enable(pVCpu);
4183	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
4184	return VINF_EM_RAW_INTERRUPT;
4185	}
4186
4187	return VINF_SUCCESS;
4188	}
4189
4190
4191	/**
4192	* Prepares to run guest (or nested-guest) code in AMD-V and we've committed to
4193	* doing so.
4194	*
4195	* This means there is no backing out to ring-3 or anywhere else at this point.
4196	*
4197	* @param pVCpu The cross context virtual CPU structure.
4198	* @param pSvmTransient Pointer to the SVM transient structure.
4199	*
4200	* @remarks Called with preemption disabled.
4201	* @remarks No-long-jump zone!!!
4202	*/
4203	static void hmR0SvmPreRunGuestCommitted(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4204	{
4205	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4206	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4207
4208	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4209	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
4210
4211	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4212	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
4213
4214	hmR0SvmInjectPendingEvent(pVCpu, pVmcb);
4215
4216	if (!CPUMIsGuestFPUStateActive(pVCpu))
4217	{
4218	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
4219	CPUMR0LoadGuestFPU(pVM, pVCpu); /* (Ignore rc, no need to set HM_CHANGED_HOST_CONTEXT for SVM.) */
4220	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
4221	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
4222	}
4223
4224	/* Load the state shared between host and guest (FPU, debug). */
4225	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)
4226	hmR0SvmExportSharedState(pVCpu, pVmcb);
4227
4228	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT; /* Preemption might set this, nothing to do on AMD-V. */
4229	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
4230
4231	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
4232	RTCPUID const idHostCpu = pHostCpu->idCpu;
4233	bool const fMigratedHostCpu = idHostCpu != pVCpu->hmr0.s.idLastCpu;
4234
4235	/* Setup TSC offsetting. */
4236	if ( pSvmTransient->fUpdateTscOffsetting
4237	\|\| fMigratedHostCpu)
4238	{
4239	hmR0SvmUpdateTscOffsetting(pVCpu, pVmcb);
4240	pSvmTransient->fUpdateTscOffsetting = false;
4241	}
4242
4243	/* Record statistics of how often we use TSC offsetting as opposed to intercepting RDTSC/P. */
4244	if (!(pVmcb->ctrl.u64InterceptCtrl & (SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP)))
4245	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4246	else
4247	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4248
4249	/* If we've migrating CPUs, mark the VMCB Clean bits as dirty. */
4250	if (fMigratedHostCpu)
4251	pVmcb->ctrl.u32VmcbCleanBits = 0;
4252
4253	/* Store status of the shared guest-host state at the time of VMRUN. */
4254	pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
4255	pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
4256
4257	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4258	uint8_t *pbMsrBitmap;
4259	if (!pSvmTransient->fIsNestedGuest)
4260	pbMsrBitmap = (uint8_t *)pVCpu->hmr0.s.svm.pvMsrBitmap;
4261	else
4262	{
4263	/** @todo We could perhaps optimize this by monitoring if the guest modifies its
4264	* MSRPM and only perform this if it changed also use EVEX.POR when it
4265	* does. */
4266	hmR0SvmMergeMsrpmNested(pHostCpu, pVCpu);
4267
4268	/* Update the nested-guest VMCB with the newly merged MSRPM (clean bits updated below). */
4269	pVmcb->ctrl.u64MSRPMPhysAddr = pHostCpu->n.svm.HCPhysNstGstMsrpm;
4270	pbMsrBitmap = (uint8_t *)pHostCpu->n.svm.pvNstGstMsrpm;
4271	}
4272	#else
4273	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.svm.pvMsrBitmap;
4274	#endif
4275
4276	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
4277	/* Flush the appropriate tagged-TLB entries. */
4278	hmR0SvmFlushTaggedTlb(pHostCpu, pVCpu, pVmcb);
4279	Assert(pVCpu->hmr0.s.idLastCpu == idHostCpu);
4280
4281	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
4282
4283	TMNotifyStartOfExecution(pVM, pVCpu); /* Finally, notify TM to resume its clocks as we're about
4284	to start executing. */
4285
4286	/*
4287	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that RDTSCPs
4288	* (that don't cause exits) reads the guest MSR, see @bugref{3324}.
4289	*
4290	* This should be done -after- any RDTSCPs for obtaining the host timestamp (TM, STAM etc).
4291	*/
4292	if ( g_CpumHostFeatures.s.fRdTscP
4293	&& !(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSCP))
4294	{
4295	uint64_t const uGuestTscAux = CPUMGetGuestTscAux(pVCpu);
4296	pVCpu->hmr0.s.svm.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
4297	if (uGuestTscAux != pVCpu->hmr0.s.svm.u64HostTscAux)
4298	ASMWrMsr(MSR_K8_TSC_AUX, uGuestTscAux);
4299	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
4300	pSvmTransient->fRestoreTscAuxMsr = true;
4301	}
4302	else
4303	{
4304	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
4305	pSvmTransient->fRestoreTscAuxMsr = false;
4306	}
4307	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
4308
4309	/*
4310	* If VMCB Clean bits isn't supported by the CPU or exposed to the guest in the nested
4311	* virtualization case, mark all state-bits as dirty indicating to the CPU to re-load
4312	* from the VMCB.
4313	*/
4314	bool const fSupportsVmcbCleanBits = hmR0SvmSupportsVmcbCleanBits(pVCpu, pSvmTransient->fIsNestedGuest);
4315	if (!fSupportsVmcbCleanBits)
4316	pVmcb->ctrl.u32VmcbCleanBits = 0;
4317	}
4318
4319
4320	/**
4321	* Wrapper for running the guest (or nested-guest) code in AMD-V.
4322	*
4323	* @returns VBox strict status code.
4324	* @param pVCpu The cross context virtual CPU structure.
4325	* @param HCPhysVmcb The host physical address of the VMCB.
4326	*
4327	* @remarks No-long-jump zone!!!
4328	*/
4329	DECLINLINE(int) hmR0SvmRunGuest(PVMCPUCC pVCpu, RTHCPHYS HCPhysVmcb)
4330	{
4331	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4332	pVCpu->cpum.GstCtx.fExtrn \|= HMSVM_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
4333	return pVCpu->hmr0.s.svm.pfnVMRun(pVCpu->CTX_SUFF(pVM), pVCpu, HCPhysVmcb);
4334	}
4335
4336
4337	/**
4338	* Performs some essential restoration of state after running guest (or
4339	* nested-guest) code in AMD-V.
4340	*
4341	* @param pVCpu The cross context virtual CPU structure.
4342	* @param pSvmTransient Pointer to the SVM transient structure.
4343	* @param rcVMRun Return code of VMRUN.
4344	*
4345	* @remarks Called with interrupts disabled.
4346	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
4347	* unconditionally when it is safe to do so.
4348	*/
4349	static void hmR0SvmPostRunGuest(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, VBOXSTRICTRC rcVMRun)
4350	{
4351	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4352
4353	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
4354	ASMAtomicIncU32(&pVCpu->hmr0.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
4355
4356	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
4357	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
4358
4359	/* TSC read must be done early for maximum accuracy. */
4360	if (!(pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSC))
4361	{
4362	if (!pSvmTransient->fIsNestedGuest)
4363	TMCpuTickSetLastSeen(pVCpu, pVCpu->hmr0.s.uTscExit + pVmcbCtrl->u64TSCOffset);
4364	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4365	else
4366	{
4367	/* The nested-guest VMCB TSC offset shall eventually be restored on #VMEXIT via HMNotifySvmNstGstVmexit(). */
4368	uint64_t const uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, pVCpu->hmr0.s.uTscExit + pVmcbCtrl->u64TSCOffset);
4369	TMCpuTickSetLastSeen(pVCpu, uGstTsc);
4370	}
4371	#endif
4372	}
4373
4374	if (pSvmTransient->fRestoreTscAuxMsr)
4375	{
4376	uint64_t u64GuestTscAuxMsr = ASMRdMsr(MSR_K8_TSC_AUX);
4377	CPUMSetGuestTscAux(pVCpu, u64GuestTscAuxMsr);
4378	if (u64GuestTscAuxMsr != pVCpu->hmr0.s.svm.u64HostTscAux)
4379	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hmr0.s.svm.u64HostTscAux);
4380	}
4381
4382	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
4383	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4384	TMNotifyEndOfExecution(pVM, pVCpu, pVCpu->hmr0.s.uTscExit); /* Notify TM that the guest is no longer running. */
4385	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4386
4387	Assert(!(ASMGetFlags() & X86_EFL_IF));
4388	ASMSetFlags(pSvmTransient->fEFlags); /* Enable interrupts. */
4389	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
4390
4391	/* If VMRUN failed, we can bail out early. This does -not- cover SVM_EXIT_INVALID. */
4392	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
4393	{
4394	Log4Func(("VMRUN failure: rcVMRun=%Rrc\n", VBOXSTRICTRC_VAL(rcVMRun)));
4395	return;
4396	}
4397
4398	pSvmTransient->u64ExitCode = pVmcbCtrl->u64ExitCode; /* Save the #VMEXIT reason. */
4399	pSvmTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
4400	pSvmTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
4401	pVmcbCtrl->u32VmcbCleanBits = HMSVM_VMCB_CLEAN_ALL; /* Mark the VMCB-state cache as unmodified by VMM. */
4402
4403	#ifdef HMSVM_SYNC_FULL_GUEST_STATE
4404	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4405	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
4406	#else
4407	/*
4408	* Always import the following:
4409	*
4410	* - RIP for exit optimizations and evaluating event injection on re-entry.
4411	* - RFLAGS for evaluating event injection on VM re-entry and for exporting shared debug
4412	* state on preemption.
4413	* - Interrupt shadow, GIF for evaluating event injection on VM re-entry.
4414	* - CS for exit optimizations.
4415	* - RAX, RSP for simplifying assumptions on GPRs. All other GPRs are swapped by the
4416	* assembly switcher code.
4417	* - Shared state (only DR7 currently) for exporting shared debug state on preemption.
4418	*/
4419	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
4420	\| CPUMCTX_EXTRN_RFLAGS
4421	\| CPUMCTX_EXTRN_RAX
4422	\| CPUMCTX_EXTRN_RSP
4423	\| CPUMCTX_EXTRN_CS
4424	\| CPUMCTX_EXTRN_HWVIRT
4425	\| CPUMCTX_EXTRN_INHIBIT_INT
4426	\| CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ
4427	\| HMSVM_CPUMCTX_SHARED_STATE);
4428	#endif
4429
4430	if ( pSvmTransient->u64ExitCode != SVM_EXIT_INVALID
4431	&& pVCpu->hmr0.s.svm.fSyncVTpr)
4432	{
4433	Assert(!pSvmTransient->fIsNestedGuest);
4434	/* TPR patching (for 32-bit guests) uses LSTAR MSR for holding the TPR value, otherwise uses the VTPR. */
4435	if ( pVM->hm.s.fTprPatchingActive
4436	&& (pVmcb->guest.u64LSTAR & 0xff) != pSvmTransient->u8GuestTpr)
4437	{
4438	int rc = APICSetTpr(pVCpu, pVmcb->guest.u64LSTAR & 0xff);
4439	AssertRC(rc);
4440	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
4441	}
4442	/* Sync TPR when we aren't intercepting CR8 writes. */
4443	else if (pSvmTransient->u8GuestTpr != pVmcbCtrl->IntCtrl.n.u8VTPR)
4444	{
4445	int rc = APICSetTpr(pVCpu, pVmcbCtrl->IntCtrl.n.u8VTPR << 4);
4446	AssertRC(rc);
4447	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
4448	}
4449	}
4450
4451	#ifdef DEBUG_ramshankar
4452	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
4453	{
4454	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4455	hmR0SvmLogState(pVCpu, pVmcb, pVCpu->cpum.GstCtx, "hmR0SvmPostRunGuestNested", HMSVM_LOG_ALL & ~HMSVM_LOG_LBR,
4456	0 /* uVerbose */);
4457	}
4458	#endif
4459
4460	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
4461	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_SVM, pSvmTransient->u64ExitCode & EMEXIT_F_TYPE_MASK),
4462	pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, pVCpu->hmr0.s.uTscExit);
4463	}
4464
4465
4466	/**
4467	* Runs the guest code using AMD-V.
4468	*
4469	* @returns Strict VBox status code.
4470	* @param pVCpu The cross context virtual CPU structure.
4471	* @param pcLoops Pointer to the number of executed loops.
4472	*/
4473	static VBOXSTRICTRC hmR0SvmRunGuestCodeNormal(PVMCPUCC pVCpu, uint32_t *pcLoops)
4474	{
4475	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops;
4476	Assert(pcLoops);
4477	Assert(*pcLoops <= cMaxResumeLoops);
4478
4479	SVMTRANSIENT SvmTransient;
4480	RT_ZERO(SvmTransient);
4481	SvmTransient.fUpdateTscOffsetting = true;
4482	SvmTransient.pVmcb = pVCpu->hmr0.s.svm.pVmcb;
4483
4484	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_5;
4485	for (;;)
4486	{
4487	Assert(!HMR0SuspendPending());
4488	HMSVM_ASSERT_CPU_SAFE(pVCpu);
4489
4490	/* Preparatory work for running nested-guest code, this may force us to return to
4491	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4492	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4493	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
4494	if (rc != VINF_SUCCESS)
4495	break;
4496
4497	/*
4498	* No longjmps to ring-3 from this point on!!!
4499	*
4500	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4501	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4502	*/
4503	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
4504	rc = hmR0SvmRunGuest(pVCpu, pVCpu->hmr0.s.svm.HCPhysVmcb);
4505
4506	/* Restore any residual host-state and save any bits shared between host and guest
4507	into the guest-CPU state. Re-enables interrupts! */
4508	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
4509
4510	if (RT_UNLIKELY( rc != VINF_SUCCESS /* Check for VMRUN errors. */
4511	\|\| SvmTransient.u64ExitCode == SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
4512	{
4513	if (rc == VINF_SUCCESS)
4514	rc = VERR_SVM_INVALID_GUEST_STATE;
4515	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
4516	hmR0SvmReportWorldSwitchError(pVCpu, VBOXSTRICTRC_VAL(rc));
4517	break;
4518	}
4519
4520	/* Handle the #VMEXIT. */
4521	HMSVM_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4522	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
4523	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, SvmTransient.u64ExitCode, pVCpu->hmr0.s.svm.pVmcb);
4524	rc = hmR0SvmHandleExit(pVCpu, &SvmTransient);
4525	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
4526	if (rc != VINF_SUCCESS)
4527	break;
4528	if (++(*pcLoops) >= cMaxResumeLoops)
4529	{
4530	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4531	rc = VINF_EM_RAW_INTERRUPT;
4532	break;
4533	}
4534	}
4535
4536	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4537	return rc;
4538	}
4539
4540
4541	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4542	/**
4543	* Runs the nested-guest code using AMD-V.
4544	*
4545	* @returns Strict VBox status code.
4546	* @param pVCpu The cross context virtual CPU structure.
4547	* @param pcLoops Pointer to the number of executed loops. If we're switching
4548	* from the guest-code execution loop to this nested-guest
4549	* execution loop pass the remainder value, else pass 0.
4550	*/
4551	static VBOXSTRICTRC hmR0SvmRunGuestCodeNested(PVMCPUCC pVCpu, uint32_t *pcLoops)
4552	{
4553	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4554	HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
4555	Assert(pcLoops);
4556	Assert(*pcLoops <= pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops);
4557	/** @todo r=bird: Sharing this with ring-3 isn't safe in the long run, I fear... */
4558	RTHCPHYS const HCPhysVmcb = GVMMR0ConvertGVMPtr2HCPhys(pVCpu->pGVM, &pCtx->hwvirt.svm.Vmcb);
4559
4560	SVMTRANSIENT SvmTransient;
4561	RT_ZERO(SvmTransient);
4562	SvmTransient.fUpdateTscOffsetting = true;
4563	SvmTransient.pVmcb = &pCtx->hwvirt.svm.Vmcb;
4564	SvmTransient.fIsNestedGuest = true;
4565
4566	/* Setup pointer so PGM/IEM can query #VMEXIT auxiliary info. on demand in ring-0. */
4567	pVCpu->hmr0.s.svm.pSvmTransient = &SvmTransient;
4568
4569	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_4;
4570	for (;;)
4571	{
4572	Assert(!HMR0SuspendPending());
4573	HMSVM_ASSERT_CPU_SAFE(pVCpu);
4574
4575	/* Preparatory work for running nested-guest code, this may force us to return to
4576	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4577	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4578	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
4579	if ( rc != VINF_SUCCESS
4580	\|\| !CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4581	break;
4582
4583	/*
4584	* No longjmps to ring-3 from this point on!!!
4585	*
4586	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4587	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4588	*/
4589	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
4590
4591	rc = hmR0SvmRunGuest(pVCpu, HCPhysVmcb);
4592
4593	/* Restore any residual host-state and save any bits shared between host and guest
4594	into the guest-CPU state. Re-enables interrupts! */
4595	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
4596
4597	if (RT_LIKELY( rc == VINF_SUCCESS
4598	&& SvmTransient.u64ExitCode != SVM_EXIT_INVALID))
4599	{ /* extremely likely */ }
4600	else
4601	{
4602	/* VMRUN failed, shouldn't really happen, Guru. */
4603	if (rc != VINF_SUCCESS)
4604	break;
4605
4606	/* Invalid nested-guest state. Cause a #VMEXIT but assert on strict builds. */
4607	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4608	AssertMsgFailed(("Invalid nested-guest state. rc=%Rrc u64ExitCode=%#RX64\n", rc, SvmTransient.u64ExitCode));
4609	rc = IEMExecSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0, 0);
4610	break;
4611	}
4612
4613	/* Handle the #VMEXIT. */
4614	HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4615	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
4616	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, &pCtx->hwvirt.svm.Vmcb);
4617	rc = hmR0SvmHandleExitNested(pVCpu, &SvmTransient);
4618	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
4619	if (rc == VINF_SUCCESS)
4620	{
4621	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4622	{
4623	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
4624	rc = VINF_SVM_VMEXIT;
4625	}
4626	else
4627	{
4628	if (++(*pcLoops) <= pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops)
4629	continue;
4630	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4631	rc = VINF_EM_RAW_INTERRUPT;
4632	}
4633	}
4634	else
4635	Assert(rc != VINF_SVM_VMEXIT);
4636	break;
4637	/** @todo NSTSVM: handle single-stepping. */
4638	}
4639
4640	/* Ensure #VMEXIT auxiliary info. is no longer available. */
4641	pVCpu->hmr0.s.svm.pSvmTransient = NULL;
4642
4643	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4644	return rc;
4645	}
4646	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
4647
4648
4649	/**
4650	* Checks if any expensive dtrace probes are enabled and we should go to the
4651	* debug loop.
4652	*
4653	* @returns true if we should use debug loop, false if not.
4654	*/
4655	static bool hmR0SvmAnyExpensiveProbesEnabled(void)
4656	{
4657	/* It's probably faster to OR the raw 32-bit counter variables together.
4658	Since the variables are in an array and the probes are next to one
4659	another (more or less), we have good locality. So, better read
4660	eight-nine cache lines ever time and only have one conditional, than
4661	128+ conditionals, right? */
4662	return ( VBOXVMM_R0_HMSVM_VMEXIT_ENABLED_RAW() /* expensive too due to context */
4663	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
4664	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
4665	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
4666	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
4667	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
4668	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
4669	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
4670	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
4671	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
4672	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
4673	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
4674	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
4675	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
4676	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
4677	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
4678	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
4679	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
4680	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
4681	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
4682	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
4683	) != 0
4684	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
4685	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
4686	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
4687	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
4688	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
4689	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
4690	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
4691	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
4692	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
4693	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
4694	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
4695	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
4696	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
4697	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
4698	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
4699	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
4700	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
4701	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
4702	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
4703	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
4704	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
4705	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
4706	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
4707	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
4708	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
4709	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
4710	//\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
4711	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
4712	//\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
4713	//\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
4714	//\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
4715	//\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
4716	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
4717	\| VBOXVMM_INSTR_SVM_VMRUN_ENABLED_RAW()
4718	\| VBOXVMM_INSTR_SVM_VMLOAD_ENABLED_RAW()
4719	\| VBOXVMM_INSTR_SVM_VMSAVE_ENABLED_RAW()
4720	\| VBOXVMM_INSTR_SVM_STGI_ENABLED_RAW()
4721	\| VBOXVMM_INSTR_SVM_CLGI_ENABLED_RAW()
4722	) != 0
4723	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
4724	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
4725	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
4726	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
4727	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
4728	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
4729	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
4730	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
4731	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
4732	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
4733	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
4734	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
4735	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
4736	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
4737	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
4738	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
4739	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
4740	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
4741	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
4742	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
4743	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
4744	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
4745	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
4746	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
4747	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
4748	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
4749	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
4750	//\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
4751	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
4752	//\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
4753	//\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
4754	//\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
4755	//\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
4756	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
4757	\| VBOXVMM_EXIT_SVM_VMRUN_ENABLED_RAW()
4758	\| VBOXVMM_EXIT_SVM_VMLOAD_ENABLED_RAW()
4759	\| VBOXVMM_EXIT_SVM_VMSAVE_ENABLED_RAW()
4760	\| VBOXVMM_EXIT_SVM_STGI_ENABLED_RAW()
4761	\| VBOXVMM_EXIT_SVM_CLGI_ENABLED_RAW()
4762	) != 0;
4763	}
4764
4765
4766	/**
4767	* Runs the guest code using AMD-V.
4768	*
4769	* @returns Strict VBox status code.
4770	* @param pVCpu The cross context virtual CPU structure.
4771	*/
4772	VMMR0DECL(VBOXSTRICTRC) SVMR0RunGuestCode(PVMCPUCC pVCpu)
4773	{
4774	AssertPtr(pVCpu);
4775	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4776	Assert(VMMRZCallRing3IsEnabled(pVCpu));
4777	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
4778	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
4779
4780	uint32_t cLoops = 0;
4781	VBOXSTRICTRC rc;
4782	for (;;)
4783	{
4784	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4785	bool const fInNestedGuestMode = CPUMIsGuestInSvmNestedHwVirtMode(pCtx);
4786	#else
4787	NOREF(pCtx);
4788	bool const fInNestedGuestMode = false;
4789	#endif
4790	if (!fInNestedGuestMode)
4791	{
4792	if ( !pVCpu->hm.s.fUseDebugLoop
4793	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0SvmAnyExpensiveProbesEnabled())
4794	&& !DBGFIsStepping(pVCpu)
4795	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
4796	rc = hmR0SvmRunGuestCodeNormal(pVCpu, &cLoops);
4797	else
4798	rc = hmR0SvmRunGuestCodeDebug(pVCpu, &cLoops);
4799	}
4800	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4801	else
4802	rc = hmR0SvmRunGuestCodeNested(pVCpu, &cLoops);
4803
4804	if (rc == VINF_SVM_VMRUN)
4805	{
4806	Assert(CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
4807	continue;
4808	}
4809	if (rc == VINF_SVM_VMEXIT)
4810	{
4811	Assert(!CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
4812	continue;
4813	}
4814	#endif
4815	break;
4816	}
4817
4818	/* Fixup error codes. */
4819	if (rc == VERR_EM_INTERPRETER)
4820	rc = VINF_EM_RAW_EMULATE_INSTR;
4821	else if (rc == VINF_EM_RESET)
4822	rc = VINF_EM_TRIPLE_FAULT;
4823
4824	/* Prepare to return to ring-3. This will remove longjmp notifications. */
4825	rc = hmR0SvmExitToRing3(pVCpu, rc);
4826	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
4827	Assert(!VMMR0AssertionIsNotificationSet(pVCpu));
4828	return rc;
4829	}
4830
4831	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4832
4833	/**
4834	* Determines whether the given I/O access should cause a nested-guest \#VMEXIT.
4835	*
4836	* @param pvIoBitmap Pointer to the nested-guest IO bitmap.
4837	* @param pIoExitInfo Pointer to the SVMIOIOEXITINFO.
4838	*/
4839	static bool hmR0SvmIsIoInterceptSet(void *pvIoBitmap, PSVMIOIOEXITINFO pIoExitInfo)
4840	{
4841	const uint16_t u16Port = pIoExitInfo->n.u16Port;
4842	const SVMIOIOTYPE enmIoType = (SVMIOIOTYPE)pIoExitInfo->n.u1Type;
4843	const uint8_t cbReg = (pIoExitInfo->u >> SVM_IOIO_OP_SIZE_SHIFT) & 7;
4844	const uint8_t cAddrSizeBits = ((pIoExitInfo->u >> SVM_IOIO_ADDR_SIZE_SHIFT) & 7) << 4;
4845	const uint8_t iEffSeg = pIoExitInfo->n.u3Seg;
4846	const bool fRep = pIoExitInfo->n.u1Rep;
4847	const bool fStrIo = pIoExitInfo->n.u1Str;
4848
4849	return CPUMIsSvmIoInterceptSet(pvIoBitmap, u16Port, enmIoType, cbReg, cAddrSizeBits, iEffSeg, fRep, fStrIo,
4850	NULL /* pIoExitInfo */);
4851	}
4852
4853
4854	/**
4855	* Handles a nested-guest \#VMEXIT (for all EXITCODE values except
4856	* SVM_EXIT_INVALID).
4857	*
4858	* @returns VBox status code (informational status codes included).
4859	* @param pVCpu The cross context virtual CPU structure.
4860	* @param pSvmTransient Pointer to the SVM transient structure.
4861	*/
4862	static VBOXSTRICTRC hmR0SvmHandleExitNested(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4863	{
4864	HMSVM_ASSERT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
4865	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
4866	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
4867
4868	/*
4869	* We import the complete state here because we use separate VMCBs for the guest and the
4870	* nested-guest, and the guest's VMCB is used after the #VMEXIT. We can only save/restore
4871	* the #VMEXIT specific state if we used the same VMCB for both guest and nested-guest.
4872	*/
4873	#define NST_GST_VMEXIT_CALL_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
4874	do { \
4875	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL); \
4876	return IEMExecSvmVmexit((a_pVCpu), (a_uExitCode), (a_uExitInfo1), (a_uExitInfo2)); \
4877	} while (0)
4878
4879	/*
4880	* For all the #VMEXITs here we primarily figure out if the #VMEXIT is expected by the
4881	* nested-guest. If it isn't, it should be handled by the (outer) guest.
4882	*/
4883	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
4884	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4885	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
4886	uint64_t const uExitCode = pVmcbNstGstCtrl->u64ExitCode;
4887	uint64_t const uExitInfo1 = pVmcbNstGstCtrl->u64ExitInfo1;
4888	uint64_t const uExitInfo2 = pVmcbNstGstCtrl->u64ExitInfo2;
4889
4890	Assert(uExitCode == pVmcbNstGstCtrl->u64ExitCode);
4891	switch (uExitCode)
4892	{
4893	case SVM_EXIT_CPUID:
4894	{
4895	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CPUID))
4896	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4897	return hmR0SvmExitCpuid(pVCpu, pSvmTransient);
4898	}
4899
4900	case SVM_EXIT_RDTSC:
4901	{
4902	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSC))
4903	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4904	return hmR0SvmExitRdtsc(pVCpu, pSvmTransient);
4905	}
4906
4907	case SVM_EXIT_RDTSCP:
4908	{
4909	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSCP))
4910	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4911	return hmR0SvmExitRdtscp(pVCpu, pSvmTransient);
4912	}
4913
4914	case SVM_EXIT_MONITOR:
4915	{
4916	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MONITOR))
4917	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4918	return hmR0SvmExitMonitor(pVCpu, pSvmTransient);
4919	}
4920
4921	case SVM_EXIT_MWAIT:
4922	{
4923	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MWAIT))
4924	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4925	return hmR0SvmExitMwait(pVCpu, pSvmTransient);
4926	}
4927
4928	case SVM_EXIT_HLT:
4929	{
4930	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_HLT))
4931	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4932	return hmR0SvmExitHlt(pVCpu, pSvmTransient);
4933	}
4934
4935	case SVM_EXIT_MSR:
4936	{
4937	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MSR_PROT))
4938	{
4939	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
4940	uint16_t offMsrpm;
4941	uint8_t uMsrpmBit;
4942	int rc = CPUMGetSvmMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
4943	if (RT_SUCCESS(rc))
4944	{
4945	Assert(uMsrpmBit == 0 \|\| uMsrpmBit == 2 \|\| uMsrpmBit == 4 \|\| uMsrpmBit == 6);
4946	Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
4947
4948	uint8_t const * const pbMsrBitmap = &pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm];
4949	bool const fInterceptRead = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit));
4950	bool const fInterceptWrite = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
4951
4952	if ( (fInterceptWrite && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_WRITE)
4953	\|\| (fInterceptRead && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_READ))
4954	{
4955	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4956	}
4957	}
4958	else
4959	{
4960	/*
4961	* MSRs not covered by the MSRPM automatically cause an #VMEXIT.
4962	* See AMD-V spec. "15.11 MSR Intercepts".
4963	*/
4964	Assert(rc == VERR_OUT_OF_RANGE);
4965	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4966	}
4967	}
4968	return hmR0SvmExitMsr(pVCpu, pSvmTransient);
4969	}
4970
4971	case SVM_EXIT_IOIO:
4972	{
4973	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IOIO_PROT))
4974	{
4975	SVMIOIOEXITINFO IoExitInfo;
4976	IoExitInfo.u = pVmcbNstGst->ctrl.u64ExitInfo1;
4977	bool const fIntercept = hmR0SvmIsIoInterceptSet(pVCpu->cpum.GstCtx.hwvirt.svm.abIoBitmap, &IoExitInfo);
4978	if (fIntercept)
4979	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4980	}
4981	return hmR0SvmExitIOInstr(pVCpu, pSvmTransient);
4982	}
4983
4984	case SVM_EXIT_XCPT_PF:
4985	{
4986	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4987	if (pVM->hmr0.s.fNestedPaging)
4988	{
4989	uint32_t const u32ErrCode = pVmcbNstGstCtrl->u64ExitInfo1;
4990	uint64_t const uFaultAddress = pVmcbNstGstCtrl->u64ExitInfo2;
4991
4992	/* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
4993	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_PF))
4994	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, u32ErrCode, uFaultAddress);
4995
4996	/* If the nested-guest is not intercepting #PFs, forward the #PF to the guest. */
4997	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR2);
4998	hmR0SvmSetPendingXcptPF(pVCpu, u32ErrCode, uFaultAddress);
4999	return VINF_SUCCESS;
5000	}
5001	return hmR0SvmExitXcptPF(pVCpu, pSvmTransient);
5002	}
5003
5004	case SVM_EXIT_XCPT_UD:
5005	{
5006	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_UD))
5007	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5008	hmR0SvmSetPendingXcptUD(pVCpu);
5009	return VINF_SUCCESS;
5010	}
5011
5012	case SVM_EXIT_XCPT_MF:
5013	{
5014	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_MF))
5015	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5016	return hmR0SvmExitXcptMF(pVCpu, pSvmTransient);
5017	}
5018
5019	case SVM_EXIT_XCPT_DB:
5020	{
5021	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_DB))
5022	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5023	return hmR0SvmNestedExitXcptDB(pVCpu, pSvmTransient);
5024	}
5025
5026	case SVM_EXIT_XCPT_AC:
5027	{
5028	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_AC))
5029	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5030	return hmR0SvmExitXcptAC(pVCpu, pSvmTransient);
5031	}
5032
5033	case SVM_EXIT_XCPT_BP:
5034	{
5035	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_BP))
5036	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5037	return hmR0SvmNestedExitXcptBP(pVCpu, pSvmTransient);
5038	}
5039
5040	case SVM_EXIT_READ_CR0:
5041	case SVM_EXIT_READ_CR3:
5042	case SVM_EXIT_READ_CR4:
5043	{
5044	uint8_t const uCr = uExitCode - SVM_EXIT_READ_CR0;
5045	if (CPUMIsGuestSvmReadCRxInterceptSet(pVCpu, pCtx, uCr))
5046	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5047	return hmR0SvmExitReadCRx(pVCpu, pSvmTransient);
5048	}
5049
5050	case SVM_EXIT_CR0_SEL_WRITE:
5051	{
5052	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CR0_SEL_WRITE))
5053	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5054	return hmR0SvmExitWriteCRx(pVCpu, pSvmTransient);
5055	}
5056
5057	case SVM_EXIT_WRITE_CR0:
5058	case SVM_EXIT_WRITE_CR3:
5059	case SVM_EXIT_WRITE_CR4:
5060	case SVM_EXIT_WRITE_CR8: /* CR8 writes would go to the V_TPR rather than here, since we run with V_INTR_MASKING. */
5061	{
5062	uint8_t const uCr = uExitCode - SVM_EXIT_WRITE_CR0;
5063	Log4Func(("Write CR%u: uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", uCr, uExitInfo1, uExitInfo2));
5064
5065	if (CPUMIsGuestSvmWriteCRxInterceptSet(pVCpu, pCtx, uCr))
5066	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5067	return hmR0SvmExitWriteCRx(pVCpu, pSvmTransient);
5068	}
5069
5070	case SVM_EXIT_PAUSE:
5071	{
5072	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_PAUSE))
5073	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5074	return hmR0SvmExitPause(pVCpu, pSvmTransient);
5075	}
5076
5077	case SVM_EXIT_VINTR:
5078	{
5079	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VINTR))
5080	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5081	return hmR0SvmExitUnexpected(pVCpu, pSvmTransient);
5082	}
5083
5084	case SVM_EXIT_INTR:
5085	case SVM_EXIT_NMI:
5086	case SVM_EXIT_SMI:
5087	case SVM_EXIT_XCPT_NMI: /* Should not occur, SVM_EXIT_NMI is used instead. */
5088	{
5089	/*
5090	* We shouldn't direct physical interrupts, NMIs, SMIs to the nested-guest.
5091	*
5092	* Although we don't intercept SMIs, the nested-guest might. Therefore, we might
5093	* get an SMI #VMEXIT here so simply ignore rather than causing a corresponding
5094	* nested-guest #VMEXIT.
5095	*
5096	* We shall import the complete state here as we may cause #VMEXITs from ring-3
5097	* while trying to inject interrupts, see comment at the top of this function.
5098	*/
5099	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_ALL);
5100	return hmR0SvmExitIntr(pVCpu, pSvmTransient);
5101	}
5102
5103	case SVM_EXIT_FERR_FREEZE:
5104	{
5105	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_FERR_FREEZE))
5106	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5107	return hmR0SvmExitFerrFreeze(pVCpu, pSvmTransient);
5108	}
5109
5110	case SVM_EXIT_INVLPG:
5111	{
5112	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPG))
5113	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5114	return hmR0SvmExitInvlpg(pVCpu, pSvmTransient);
5115	}
5116
5117	case SVM_EXIT_WBINVD:
5118	{
5119	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_WBINVD))
5120	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5121	return hmR0SvmExitWbinvd(pVCpu, pSvmTransient);
5122	}
5123
5124	case SVM_EXIT_INVD:
5125	{
5126	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVD))
5127	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5128	return hmR0SvmExitInvd(pVCpu, pSvmTransient);
5129	}
5130
5131	case SVM_EXIT_RDPMC:
5132	{
5133	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDPMC))
5134	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5135	return hmR0SvmExitRdpmc(pVCpu, pSvmTransient);
5136	}
5137
5138	default:
5139	{
5140	switch (uExitCode)
5141	{
5142	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5143	case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5144	case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5145	case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5146	{
5147	uint8_t const uDr = uExitCode - SVM_EXIT_READ_DR0;
5148	if (CPUMIsGuestSvmReadDRxInterceptSet(pVCpu, pCtx, uDr))
5149	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5150	return hmR0SvmExitReadDRx(pVCpu, pSvmTransient);
5151	}
5152
5153	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5154	case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5155	case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5156	case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5157	{
5158	uint8_t const uDr = uExitCode - SVM_EXIT_WRITE_DR0;
5159	if (CPUMIsGuestSvmWriteDRxInterceptSet(pVCpu, pCtx, uDr))
5160	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5161	return hmR0SvmExitWriteDRx(pVCpu, pSvmTransient);
5162	}
5163
5164	case SVM_EXIT_XCPT_DE:
5165	/* SVM_EXIT_XCPT_DB: / / Handled above. */
5166	/* SVM_EXIT_XCPT_NMI: / / Handled above. */
5167	/* SVM_EXIT_XCPT_BP: / / Handled above. */
5168	case SVM_EXIT_XCPT_OF:
5169	case SVM_EXIT_XCPT_BR:
5170	/* SVM_EXIT_XCPT_UD: / / Handled above. */
5171	case SVM_EXIT_XCPT_NM:
5172	case SVM_EXIT_XCPT_DF:
5173	case SVM_EXIT_XCPT_CO_SEG_OVERRUN:
5174	case SVM_EXIT_XCPT_TS:
5175	case SVM_EXIT_XCPT_NP:
5176	case SVM_EXIT_XCPT_SS:
5177	case SVM_EXIT_XCPT_GP:
5178	/* SVM_EXIT_XCPT_PF: / / Handled above. */
5179	case SVM_EXIT_XCPT_15: /* Reserved. */
5180	/* SVM_EXIT_XCPT_MF: / / Handled above. */
5181	/* SVM_EXIT_XCPT_AC: / / Handled above. */
5182	case SVM_EXIT_XCPT_MC:
5183	case SVM_EXIT_XCPT_XF:
5184	case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
5185	case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
5186	case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
5187	{
5188	uint8_t const uVector = uExitCode - SVM_EXIT_XCPT_0;
5189	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, uVector))
5190	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5191	return hmR0SvmExitXcptGeneric(pVCpu, pSvmTransient);
5192	}
5193
5194	case SVM_EXIT_XSETBV:
5195	{
5196	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_XSETBV))
5197	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5198	return hmR0SvmExitXsetbv(pVCpu, pSvmTransient);
5199	}
5200
5201	case SVM_EXIT_TASK_SWITCH:
5202	{
5203	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_TASK_SWITCH))
5204	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5205	return hmR0SvmExitTaskSwitch(pVCpu, pSvmTransient);
5206	}
5207
5208	case SVM_EXIT_IRET:
5209	{
5210	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IRET))
5211	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5212	return hmR0SvmExitIret(pVCpu, pSvmTransient);
5213	}
5214
5215	case SVM_EXIT_SHUTDOWN:
5216	{
5217	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SHUTDOWN))
5218	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5219	return hmR0SvmExitShutdown(pVCpu, pSvmTransient);
5220	}
5221
5222	case SVM_EXIT_VMMCALL:
5223	{
5224	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMMCALL))
5225	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5226	return hmR0SvmExitVmmCall(pVCpu, pSvmTransient);
5227	}
5228
5229	case SVM_EXIT_CLGI:
5230	{
5231	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CLGI))
5232	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5233	return hmR0SvmExitClgi(pVCpu, pSvmTransient);
5234	}
5235
5236	case SVM_EXIT_STGI:
5237	{
5238	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_STGI))
5239	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5240	return hmR0SvmExitStgi(pVCpu, pSvmTransient);
5241	}
5242
5243	case SVM_EXIT_VMLOAD:
5244	{
5245	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMLOAD))
5246	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5247	return hmR0SvmExitVmload(pVCpu, pSvmTransient);
5248	}
5249
5250	case SVM_EXIT_VMSAVE:
5251	{
5252	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMSAVE))
5253	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5254	return hmR0SvmExitVmsave(pVCpu, pSvmTransient);
5255	}
5256
5257	case SVM_EXIT_INVLPGA:
5258	{
5259	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPGA))
5260	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5261	return hmR0SvmExitInvlpga(pVCpu, pSvmTransient);
5262	}
5263
5264	case SVM_EXIT_VMRUN:
5265	{
5266	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMRUN))
5267	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5268	return hmR0SvmExitVmrun(pVCpu, pSvmTransient);
5269	}
5270
5271	case SVM_EXIT_RSM:
5272	{
5273	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RSM))
5274	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5275	hmR0SvmSetPendingXcptUD(pVCpu);
5276	return VINF_SUCCESS;
5277	}
5278
5279	case SVM_EXIT_SKINIT:
5280	{
5281	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SKINIT))
5282	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5283	hmR0SvmSetPendingXcptUD(pVCpu);
5284	return VINF_SUCCESS;
5285	}
5286
5287	case SVM_EXIT_NPF:
5288	{
5289	Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
5290	return hmR0SvmExitNestedPF(pVCpu, pSvmTransient);
5291	}
5292
5293	case SVM_EXIT_INIT: /* We shouldn't get INIT signals while executing a nested-guest. */
5294	return hmR0SvmExitUnexpected(pVCpu, pSvmTransient);
5295
5296	default:
5297	{
5298	AssertMsgFailed(("hmR0SvmHandleExitNested: Unknown exit code %#x\n", pSvmTransient->u64ExitCode));
5299	pVCpu->hm.s.u32HMError = pSvmTransient->u64ExitCode;
5300	return VERR_SVM_UNKNOWN_EXIT;
5301	}
5302	}
5303	}
5304	}
5305	/* not reached */
5306
5307	# undef NST_GST_VMEXIT_CALL_RET
5308	}
5309
5310	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
5311
5312	/** @def VMEXIT_CALL_RET
5313	* Used by hmR0SvmHandleExit and hmR0SvmDebugHandleExit
5314	*/
5315	#ifdef DEBUG_ramshankar
5316	# define VMEXIT_CALL_RET(a_fDbg, a_CallExpr) \
5317	do { \
5318	if ((a_fDbg) == 1) \
5319	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL); \
5320	int rc = a_CallExpr; \
5321	if ((a_fDbg) == 1) \
5322	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
5323	return rc; \
5324	} while (0)
5325	#else
5326	# define VMEXIT_CALL_RET(a_fDbg, a_CallExpr) return a_CallExpr
5327	#endif
5328
5329	/**
5330	* Handles a guest \#VMEXIT (for all EXITCODE values except SVM_EXIT_INVALID).
5331	*
5332	* @returns Strict VBox status code (informational status codes included).
5333	* @param pVCpu The cross context virtual CPU structure.
5334	* @param pSvmTransient Pointer to the SVM transient structure.
5335	*/
5336	static VBOXSTRICTRC hmR0SvmHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
5337	{
5338	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
5339	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
5340
5341	/*
5342	* The ordering of the case labels is based on most-frequently-occurring #VMEXITs
5343	* for most guests under normal workloads (for some definition of "normal").
5344	*/
5345	uint64_t const uExitCode = pSvmTransient->u64ExitCode;
5346	switch (uExitCode)
5347	{
5348	case SVM_EXIT_NPF: VMEXIT_CALL_RET(0, hmR0SvmExitNestedPF(pVCpu, pSvmTransient));
5349	case SVM_EXIT_IOIO: VMEXIT_CALL_RET(0, hmR0SvmExitIOInstr(pVCpu, pSvmTransient));
5350	case SVM_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0SvmExitRdtsc(pVCpu, pSvmTransient));
5351	case SVM_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0SvmExitRdtscp(pVCpu, pSvmTransient));
5352	case SVM_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0SvmExitCpuid(pVCpu, pSvmTransient));
5353	case SVM_EXIT_XCPT_PF: VMEXIT_CALL_RET(0, hmR0SvmExitXcptPF(pVCpu, pSvmTransient));
5354	case SVM_EXIT_MSR: VMEXIT_CALL_RET(0, hmR0SvmExitMsr(pVCpu, pSvmTransient));
5355	case SVM_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0SvmExitMonitor(pVCpu, pSvmTransient));
5356	case SVM_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0SvmExitMwait(pVCpu, pSvmTransient));
5357	case SVM_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0SvmExitHlt(pVCpu, pSvmTransient));
5358
5359	case SVM_EXIT_XCPT_NMI: /* Should not occur, SVM_EXIT_NMI is used instead. */
5360	case SVM_EXIT_INTR:
5361	case SVM_EXIT_NMI: VMEXIT_CALL_RET(0, hmR0SvmExitIntr(pVCpu, pSvmTransient));
5362
5363	case SVM_EXIT_READ_CR0:
5364	case SVM_EXIT_READ_CR3:
5365	case SVM_EXIT_READ_CR4: VMEXIT_CALL_RET(0, hmR0SvmExitReadCRx(pVCpu, pSvmTransient));
5366
5367	case SVM_EXIT_CR0_SEL_WRITE:
5368	case SVM_EXIT_WRITE_CR0:
5369	case SVM_EXIT_WRITE_CR3:
5370	case SVM_EXIT_WRITE_CR4:
5371	case SVM_EXIT_WRITE_CR8: VMEXIT_CALL_RET(0, hmR0SvmExitWriteCRx(pVCpu, pSvmTransient));
5372
5373	case SVM_EXIT_VINTR: VMEXIT_CALL_RET(0, hmR0SvmExitVIntr(pVCpu, pSvmTransient));
5374	case SVM_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0SvmExitPause(pVCpu, pSvmTransient));
5375	case SVM_EXIT_VMMCALL: VMEXIT_CALL_RET(0, hmR0SvmExitVmmCall(pVCpu, pSvmTransient));
5376	case SVM_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0SvmExitInvlpg(pVCpu, pSvmTransient));
5377	case SVM_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0SvmExitWbinvd(pVCpu, pSvmTransient));
5378	case SVM_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0SvmExitInvd(pVCpu, pSvmTransient));
5379	case SVM_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0SvmExitRdpmc(pVCpu, pSvmTransient));
5380	case SVM_EXIT_IRET: VMEXIT_CALL_RET(0, hmR0SvmExitIret(pVCpu, pSvmTransient));
5381	case SVM_EXIT_XCPT_DE: VMEXIT_CALL_RET(0, hmR0SvmExitXcptDE(pVCpu, pSvmTransient));
5382	case SVM_EXIT_XCPT_UD: VMEXIT_CALL_RET(0, hmR0SvmExitXcptUD(pVCpu, pSvmTransient));
5383	case SVM_EXIT_XCPT_MF: VMEXIT_CALL_RET(0, hmR0SvmExitXcptMF(pVCpu, pSvmTransient));
5384	case SVM_EXIT_XCPT_DB: VMEXIT_CALL_RET(0, hmR0SvmExitXcptDB(pVCpu, pSvmTransient));
5385	case SVM_EXIT_XCPT_AC: VMEXIT_CALL_RET(0, hmR0SvmExitXcptAC(pVCpu, pSvmTransient));
5386	case SVM_EXIT_XCPT_BP: VMEXIT_CALL_RET(0, hmR0SvmExitXcptBP(pVCpu, pSvmTransient));
5387	case SVM_EXIT_XCPT_GP: VMEXIT_CALL_RET(0, hmR0SvmExitXcptGP(pVCpu, pSvmTransient));
5388	case SVM_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0SvmExitXsetbv(pVCpu, pSvmTransient));
5389	case SVM_EXIT_FERR_FREEZE: VMEXIT_CALL_RET(0, hmR0SvmExitFerrFreeze(pVCpu, pSvmTransient));
5390
5391	default:
5392	{
5393	switch (pSvmTransient->u64ExitCode)
5394	{
5395	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5396	case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5397	case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5398	case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5399	VMEXIT_CALL_RET(0, hmR0SvmExitReadDRx(pVCpu, pSvmTransient));
5400
5401	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5402	case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5403	case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5404	case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5405	VMEXIT_CALL_RET(0, hmR0SvmExitWriteDRx(pVCpu, pSvmTransient));
5406
5407	case SVM_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0SvmExitTaskSwitch(pVCpu, pSvmTransient));
5408	case SVM_EXIT_SHUTDOWN: VMEXIT_CALL_RET(0, hmR0SvmExitShutdown(pVCpu, pSvmTransient));
5409
5410	case SVM_EXIT_SMI:
5411	case SVM_EXIT_INIT:
5412	{
5413	/*
5414	* We don't intercept SMIs. As for INIT signals, it really shouldn't ever happen here.
5415	* If it ever does, we want to know about it so log the exit code and bail.
5416	*/
5417	VMEXIT_CALL_RET(0, hmR0SvmExitUnexpected(pVCpu, pSvmTransient));
5418	}
5419
5420	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
5421	case SVM_EXIT_CLGI: VMEXIT_CALL_RET(0, hmR0SvmExitClgi(pVCpu, pSvmTransient));
5422	case SVM_EXIT_STGI: VMEXIT_CALL_RET(0, hmR0SvmExitStgi(pVCpu, pSvmTransient));
5423	case SVM_EXIT_VMLOAD: VMEXIT_CALL_RET(0, hmR0SvmExitVmload(pVCpu, pSvmTransient));
5424	case SVM_EXIT_VMSAVE: VMEXIT_CALL_RET(0, hmR0SvmExitVmsave(pVCpu, pSvmTransient));
5425	case SVM_EXIT_INVLPGA: VMEXIT_CALL_RET(0, hmR0SvmExitInvlpga(pVCpu, pSvmTransient));
5426	case SVM_EXIT_VMRUN: VMEXIT_CALL_RET(0, hmR0SvmExitVmrun(pVCpu, pSvmTransient));
5427	#else
5428	case SVM_EXIT_CLGI:
5429	case SVM_EXIT_STGI:
5430	case SVM_EXIT_VMLOAD:
5431	case SVM_EXIT_VMSAVE:
5432	case SVM_EXIT_INVLPGA:
5433	case SVM_EXIT_VMRUN:
5434	#endif
5435	case SVM_EXIT_RSM:
5436	case SVM_EXIT_SKINIT:
5437	{
5438	hmR0SvmSetPendingXcptUD(pVCpu);
5439	return VINF_SUCCESS;
5440	}
5441
5442	/*
5443	* The remaining should only be possible when debugging or dtracing.
5444	*/
5445	case SVM_EXIT_XCPT_DE:
5446	/* SVM_EXIT_XCPT_DB: / / Handled above. */
5447	/* SVM_EXIT_XCPT_NMI: / / Handled above. */
5448	/* SVM_EXIT_XCPT_BP: / / Handled above. */
5449	case SVM_EXIT_XCPT_OF:
5450	case SVM_EXIT_XCPT_BR:
5451	/* SVM_EXIT_XCPT_UD: / / Handled above. */
5452	case SVM_EXIT_XCPT_NM:
5453	case SVM_EXIT_XCPT_DF:
5454	case SVM_EXIT_XCPT_CO_SEG_OVERRUN:
5455	case SVM_EXIT_XCPT_TS:
5456	case SVM_EXIT_XCPT_NP:
5457	case SVM_EXIT_XCPT_SS:
5458	/* SVM_EXIT_XCPT_GP: / / Handled above. */
5459	/* SVM_EXIT_XCPT_PF: */
5460	case SVM_EXIT_XCPT_15: /* Reserved. */
5461	/* SVM_EXIT_XCPT_MF: / / Handled above. */
5462	/* SVM_EXIT_XCPT_AC: / / Handled above. */
5463	case SVM_EXIT_XCPT_MC:
5464	case SVM_EXIT_XCPT_XF:
5465	case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
5466	case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
5467	case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
5468	VMEXIT_CALL_RET(0, hmR0SvmExitXcptGeneric(pVCpu, pSvmTransient));
5469
5470	case SVM_EXIT_SWINT: VMEXIT_CALL_RET(0, hmR0SvmExitSwInt(pVCpu, pSvmTransient));
5471	case SVM_EXIT_TR_READ: VMEXIT_CALL_RET(0, hmR0SvmExitTrRead(pVCpu, pSvmTransient));
5472	case SVM_EXIT_TR_WRITE: VMEXIT_CALL_RET(0, hmR0SvmExitTrWrite(pVCpu, pSvmTransient)); /* Also OS/2 TLB workaround. */
5473
5474	default:
5475	{
5476	AssertMsgFailed(("hmR0SvmHandleExit: Unknown exit code %#RX64\n", uExitCode));
5477	pVCpu->hm.s.u32HMError = uExitCode;
5478	return VERR_SVM_UNKNOWN_EXIT;
5479	}
5480	}
5481	}
5482	}
5483	/* not reached */
5484	}
5485
5486
5487	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace probes.
5488	*
5489	* The following few functions and associated structure contains the bloat
5490	* necessary for providing detailed debug events and dtrace probes as well as
5491	* reliable host side single stepping. This works on the principle of
5492	* "subclassing" the normal execution loop and workers. We replace the loop
5493	* method completely and override selected helpers to add necessary adjustments
5494	* to their core operation.
5495	*
5496	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
5497	* any performance for debug and analysis features.
5498	*
5499	* @{
5500	*/
5501
5502	/**
5503	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
5504	* the debug run loop.
5505	*/
5506	typedef struct SVMRUNDBGSTATE
5507	{
5508	/** The initial SVMVMCBCTRL::u64InterceptCtrl value (helps with restore). */
5509	uint64_t bmInterceptInitial;
5510	/** The initial SVMVMCBCTRL::u32InterceptXcpt value (helps with restore). */
5511	uint32_t bmXcptInitial;
5512	/** The initial SVMVMCBCTRL::u16InterceptRdCRx value (helps with restore). */
5513	uint16_t bmInterceptRdCRxInitial;
5514	/** The initial SVMVMCBCTRL::u16InterceptWrCRx value (helps with restore). */
5515	uint16_t bmInterceptWrCRxInitial;
5516	/** The initial SVMVMCBCTRL::u16InterceptRdDRx value (helps with restore). */
5517	uint16_t bmInterceptRdDRxInitial;
5518	/** The initial SVMVMCBCTRL::u16InterceptWrDRx value (helps with restore). */
5519	uint16_t bmInterceptWrDRxInitial;
5520
5521	/** Whether we've actually modified the intercept control qword. */
5522	bool fModifiedInterceptCtrl : 1;
5523	/** Whether we've actually modified the exception bitmap. */
5524	bool fModifiedXcptBitmap : 1;
5525	/** Whether we've actually modified SVMVMCBCTRL::u16InterceptRdCRx. */
5526	bool fModifiedInterceptRdCRx : 1;
5527	/** Whether we've actually modified SVMVMCBCTRL::u16InterceptWrCRx. */
5528	bool fModifiedInterceptWrCRx : 1;
5529	/** Whether we've actually modified SVMVMCBCTRL::u16InterceptRdDRx. */
5530	bool fModifiedInterceptRdDRx : 1;
5531	/** Whether we've actually modified SVMVMCBCTRL::u16InterceptWrDRx. */
5532	bool fModifiedInterceptWrDRx : 1;
5533
5534	/** The CS we started executing with. */
5535	uint16_t uCsStart;
5536	/** The RIP we started executing at. This is for detecting that we stepped. */
5537	uint64_t uRipStart;
5538
5539	/** The sequence number of the Dtrace provider settings the state was
5540	* configured against. */
5541	uint32_t uDtraceSettingsSeqNo;
5542	/** Extra stuff we need in SVMVMCBCTRL::u32InterceptXcpt. */
5543	uint32_t bmXcptExtra;
5544	/** Extra stuff we need in SVMVMCBCTRL::u64InterceptCtrl. */
5545	uint64_t bmInterceptExtra;
5546	/** Extra stuff we need in SVMVMCBCTRL::u16InterceptRdCRx. */
5547	uint16_t bmInterceptRdCRxExtra;
5548	/** Extra stuff we need in SVMVMCBCTRL::u16InterceptWrCRx. */
5549	uint16_t bmInterceptWrCRxExtra;
5550	/** Extra stuff we need in SVMVMCBCTRL::u16InterceptRdDRx. */
5551	uint16_t bmInterceptRdDRxExtra;
5552	/** Extra stuff we need in SVMVMCBCTRL::u16InterceptWrDRx. */
5553	uint16_t bmInterceptWrDRxExtra;
5554	/** VM-exits to check (one bit per VM-exit). */
5555	uint32_t bmExitsToCheck[33];
5556	} SVMRUNDBGSTATE;
5557	AssertCompileMemberSize(SVMRUNDBGSTATE, bmExitsToCheck, (SVM_EXIT_MAX + 1 + 31) / 32 * 4);
5558	typedef SVMRUNDBGSTATE *PSVMRUNDBGSTATE;
5559
5560
5561	/**
5562	* Initializes the SVMRUNDBGSTATE structure.
5563	*
5564	* @param pVCpu The cross context virtual CPU structure of the
5565	* calling EMT.
5566	* @param pSvmTransient The SVM-transient structure.
5567	* @param pDbgState The debug state to initialize.
5568	*/
5569	static void hmR0SvmRunDebugStateInit(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5570	{
5571	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
5572	pDbgState->bmInterceptInitial = pVmcb->ctrl.u64InterceptCtrl;
5573	pDbgState->bmXcptInitial = pVmcb->ctrl.u32InterceptXcpt;
5574	pDbgState->bmInterceptRdCRxInitial = pVmcb->ctrl.u16InterceptRdCRx;
5575	pDbgState->bmInterceptWrCRxInitial = pVmcb->ctrl.u16InterceptWrCRx;
5576	pDbgState->bmInterceptRdDRxInitial = pVmcb->ctrl.u16InterceptRdDRx;
5577	pDbgState->bmInterceptWrDRxInitial = pVmcb->ctrl.u16InterceptWrDRx;
5578
5579	pDbgState->fModifiedInterceptCtrl = false;
5580	pDbgState->fModifiedXcptBitmap = false;
5581	pDbgState->fModifiedInterceptRdCRx = false;
5582	pDbgState->fModifiedInterceptWrCRx = false;
5583	pDbgState->fModifiedInterceptRdDRx = false;
5584	pDbgState->fModifiedInterceptWrDRx = false;
5585
5586	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
5587	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
5588
5589	/* We don't really need to zero these. */
5590	pDbgState->bmInterceptExtra = 0;
5591	pDbgState->bmXcptExtra = 0;
5592	pDbgState->bmInterceptRdCRxExtra = 0;
5593	pDbgState->bmInterceptWrCRxExtra = 0;
5594	pDbgState->bmInterceptRdDRxExtra = 0;
5595	pDbgState->bmInterceptWrDRxExtra = 0;
5596	}
5597
5598
5599	/**
5600	* Updates the VMCB fields with changes requested by @a pDbgState.
5601	*
5602	* This is performed after hmR0SvmPreRunGuestDebugStateUpdate as well
5603	* immediately before executing guest code, i.e. when interrupts are disabled.
5604	* We don't check status codes here as we cannot easily assert or return in the
5605	* latter case.
5606	*
5607	* @param pSvmTransient The SVM-transient structure.
5608	* @param pDbgState The debug state.
5609	*/
5610	static void hmR0SvmPreRunGuestDebugStateApply(PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5611	{
5612	/*
5613	* Ensure desired flags in VMCS control fields are set.
5614	*/
5615	PSVMVMCB const pVmcb = pSvmTransient->pVmcb;
5616	#define ADD_EXTRA_INTERCEPTS(a_VmcbCtrlField, a_bmExtra, a_fModified) do { \
5617	if ((pVmcb->ctrl. a_VmcbCtrlField & (a_bmExtra)) != (a_bmExtra)) \
5618	{ \
5619	pVmcb->ctrl. a_VmcbCtrlField \|= (a_bmExtra); \
5620	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS; \
5621	Log6Func((#a_VmcbCtrlField ": %#RX64\n", pVmcb->ctrl. a_VmcbCtrlField)); \
5622	(a_fModified) = true; \
5623	} \
5624	} while (0)
5625	ADD_EXTRA_INTERCEPTS(u64InterceptCtrl, pDbgState->bmInterceptExtra, pDbgState->fModifiedInterceptCtrl);
5626	ADD_EXTRA_INTERCEPTS(u32InterceptXcpt, pDbgState->bmXcptExtra, pDbgState->fModifiedXcptBitmap);
5627	ADD_EXTRA_INTERCEPTS(u16InterceptRdCRx, pDbgState->bmInterceptRdCRxExtra, pDbgState->fModifiedInterceptRdCRx);
5628	ADD_EXTRA_INTERCEPTS(u16InterceptWrCRx, pDbgState->bmInterceptWrCRxExtra, pDbgState->fModifiedInterceptWrCRx);
5629	ADD_EXTRA_INTERCEPTS(u16InterceptRdDRx, pDbgState->bmInterceptRdDRxExtra, pDbgState->fModifiedInterceptRdDRx);
5630	ADD_EXTRA_INTERCEPTS(u16InterceptWrDRx, pDbgState->bmInterceptWrDRxExtra, pDbgState->fModifiedInterceptWrDRx);
5631	#undef ADD_EXTRA_INTERCEPTS
5632	}
5633
5634
5635	/**
5636	* Restores VMCB fields that were changed by hmR0SvmPreRunGuestDebugStateApply
5637	* for re-entry next time around.
5638	*
5639	* @returns Strict VBox status code (i.e. informational status codes too).
5640	* @param pSvmTransient The SVM-transient structure.
5641	* @param pDbgState The debug state.
5642	*/
5643	static void hmR0SvmRunDebugStateRevert(PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5644	{
5645	/*
5646	* Restore VM-exit control settings as we may not reenter this function the
5647	* next time around.
5648	*/
5649	PSVMVMCB const pVmcb = pSvmTransient->pVmcb;
5650
5651	#define RESTORE_INTERCEPTS(a_VmcbCtrlField, a_bmInitial, a_fModified) do { \
5652	if ((a_fModified)) \
5653	{ \
5654	pVmcb->ctrl. a_VmcbCtrlField = (a_bmInitial); \
5655	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS; \
5656	} \
5657	} while (0)
5658	RESTORE_INTERCEPTS(u64InterceptCtrl, pDbgState->bmInterceptInitial, pDbgState->fModifiedInterceptCtrl);
5659	RESTORE_INTERCEPTS(u32InterceptXcpt, pDbgState->bmXcptInitial, pDbgState->fModifiedXcptBitmap);
5660	RESTORE_INTERCEPTS(u16InterceptRdCRx, pDbgState->bmInterceptRdCRxInitial, pDbgState->fModifiedInterceptRdCRx);
5661	RESTORE_INTERCEPTS(u16InterceptWrCRx, pDbgState->bmInterceptWrCRxInitial, pDbgState->fModifiedInterceptWrCRx);
5662	RESTORE_INTERCEPTS(u16InterceptRdDRx, pDbgState->bmInterceptRdDRxInitial, pDbgState->fModifiedInterceptRdDRx);
5663	RESTORE_INTERCEPTS(u16InterceptWrDRx, pDbgState->bmInterceptWrDRxInitial, pDbgState->fModifiedInterceptWrDRx);
5664	#undef RESTORE_INTERCEPTS
5665	}
5666
5667
5668	/**
5669	* Configures VM-exit controls for current DBGF and DTrace settings.
5670	*
5671	* This updates @a pDbgState and the VMCB execution control fields (in the debug
5672	* state) to reflect the necessary VM-exits demanded by DBGF and DTrace.
5673	*
5674	* @param pVCpu The cross context virtual CPU structure.
5675	* @param pSvmTransient The SVM-transient structure. May update
5676	* fUpdatedTscOffsettingAndPreemptTimer.
5677	* @param pDbgState The debug state.
5678	*/
5679	static void hmR0SvmPreRunGuestDebugStateUpdate(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5680	{
5681	/*
5682	* Take down the dtrace serial number so we can spot changes.
5683	*/
5684	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
5685	ASMCompilerBarrier();
5686
5687	/*
5688	* Clear data members that we'll be rebuilding here.
5689	*/
5690	pDbgState->bmXcptExtra = 0;
5691	pDbgState->bmInterceptExtra = 0;
5692	pDbgState->bmInterceptRdCRxExtra = 0;
5693	pDbgState->bmInterceptWrCRxExtra = 0;
5694	pDbgState->bmInterceptRdDRxExtra = 0;
5695	pDbgState->bmInterceptWrDRxExtra = 0;
5696	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
5697	pDbgState->bmExitsToCheck[i] = 0;
5698
5699	/*
5700	* Software interrupts (INT XXh)
5701	*/
5702	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5703	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
5704	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
5705	{
5706	pDbgState->bmInterceptExtra \|= SVM_CTRL_INTERCEPT_INTN;
5707	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_SWINT);
5708	}
5709
5710	/*
5711	* INT3 breakpoints - triggered by #BP exceptions.
5712	*/
5713	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
5714	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
5715
5716	/*
5717	* Exception bitmap and XCPT events+probes.
5718	*/
5719	#define SET_XCPT(a_iXcpt) do { \
5720	pDbgState->bmXcptExtra \|= RT_BIT_32(a_iXcpt); \
5721	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_XCPT_0 + (a_iXcpt)); \
5722	} while (0)
5723
5724	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
5725	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
5726	SET_XCPT(iXcpt);
5727
5728	if (VBOXVMM_XCPT_DE_ENABLED()) SET_XCPT(X86_XCPT_DE);
5729	if (VBOXVMM_XCPT_DB_ENABLED()) SET_XCPT(X86_XCPT_DB);
5730	if (VBOXVMM_XCPT_BP_ENABLED()) SET_XCPT(X86_XCPT_BP);
5731	if (VBOXVMM_XCPT_OF_ENABLED()) SET_XCPT(X86_XCPT_OF);
5732	if (VBOXVMM_XCPT_BR_ENABLED()) SET_XCPT(X86_XCPT_BR);
5733	if (VBOXVMM_XCPT_UD_ENABLED()) SET_XCPT(X86_XCPT_UD);
5734	if (VBOXVMM_XCPT_NM_ENABLED()) SET_XCPT(X86_XCPT_NM);
5735	if (VBOXVMM_XCPT_DF_ENABLED()) SET_XCPT(X86_XCPT_DF);
5736	if (VBOXVMM_XCPT_TS_ENABLED()) SET_XCPT(X86_XCPT_TS);
5737	if (VBOXVMM_XCPT_NP_ENABLED()) SET_XCPT(X86_XCPT_NP);
5738	if (VBOXVMM_XCPT_SS_ENABLED()) SET_XCPT(X86_XCPT_SS);
5739	if (VBOXVMM_XCPT_GP_ENABLED()) SET_XCPT(X86_XCPT_GP);
5740	if (VBOXVMM_XCPT_PF_ENABLED()) SET_XCPT(X86_XCPT_PF);
5741	if (VBOXVMM_XCPT_MF_ENABLED()) SET_XCPT(X86_XCPT_MF);
5742	if (VBOXVMM_XCPT_AC_ENABLED()) SET_XCPT(X86_XCPT_AC);
5743	if (VBOXVMM_XCPT_XF_ENABLED()) SET_XCPT(X86_XCPT_XF);
5744	if (VBOXVMM_XCPT_VE_ENABLED()) SET_XCPT(X86_XCPT_VE);
5745	if (VBOXVMM_XCPT_SX_ENABLED()) SET_XCPT(X86_XCPT_SX);
5746
5747	#undef SET_XCPT
5748
5749	/*
5750	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
5751	*
5752	* Note! This is the reverse of what hmR0SvmHandleExitDtraceEvents does.
5753	* So, when adding/changing/removing please don't forget to update it.
5754	*
5755	* Some of the macros are picking up local variables to save horizontal space,
5756	* (being able to see it in a table is the lesser evil here).
5757	*/
5758	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
5759	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
5760	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
5761	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
5762	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
5763	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
5764	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
5765	} else do { } while (0)
5766	#define SET_INCP_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fInterceptCtrl) \
5767	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
5768	{ \
5769	(pDbgState)->bmInterceptExtra \|= (a_fInterceptCtrl); \
5770	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
5771	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
5772	} else do { } while (0)
5773
5774	/** @todo double check these */
5775	/** @todo Check what more AMD-V specific we can intercept. */
5776	//SET_INCP_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, SVM_EXIT_TASK_SWITCH, SVM_CTRL_INTERCEPT_TASK_SWITCH);
5777	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, SVM_EXIT_TASK_SWITCH);
5778	SET_INCP_XBM_IF_EITHER_EN(INSTR_VMM_CALL, SVM_EXIT_VMMCALL, SVM_CTRL_INTERCEPT_VMMCALL);
5779	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, SVM_EXIT_VMMCALL);
5780	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_VMRUN, SVM_EXIT_VMRUN, SVM_CTRL_INTERCEPT_VMRUN);
5781	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_VMRUN, SVM_EXIT_VMRUN);
5782	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_VMLOAD, SVM_EXIT_VMLOAD, SVM_CTRL_INTERCEPT_VMLOAD);
5783	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_VMLOAD, SVM_EXIT_VMLOAD);
5784	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_VMSAVE, SVM_EXIT_VMSAVE, SVM_CTRL_INTERCEPT_VMSAVE);
5785	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_VMSAVE, SVM_EXIT_VMSAVE);
5786	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_STGI, SVM_EXIT_STGI, SVM_CTRL_INTERCEPT_STGI);
5787	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_STGI, SVM_EXIT_STGI);
5788	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_CLGI, SVM_EXIT_CLGI, SVM_CTRL_INTERCEPT_CLGI);
5789	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_CLGI, SVM_EXIT_CLGI);
5790
5791	SET_INCP_XBM_IF_EITHER_EN(INSTR_CPUID, SVM_EXIT_CPUID, SVM_CTRL_INTERCEPT_CPUID);
5792	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, SVM_EXIT_CPUID);
5793	SET_INCP_XBM_IF_EITHER_EN(INSTR_HALT, SVM_EXIT_HLT, SVM_CTRL_INTERCEPT_HLT);
5794	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, SVM_EXIT_HLT);
5795	SET_INCP_XBM_IF_EITHER_EN(INSTR_INVD, SVM_EXIT_INVD, SVM_CTRL_INTERCEPT_INVD);
5796	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, SVM_EXIT_INVD);
5797	SET_INCP_XBM_IF_EITHER_EN(INSTR_INVLPG, SVM_EXIT_INVLPG, SVM_CTRL_INTERCEPT_INVLPG);
5798	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, SVM_EXIT_INVLPG);
5799	SET_INCP_XBM_IF_EITHER_EN(INSTR_RDPMC, SVM_EXIT_RDPMC, SVM_CTRL_INTERCEPT_RDPMC);
5800	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, SVM_EXIT_RDPMC);
5801	SET_INCP_XBM_IF_EITHER_EN(INSTR_RDTSC, SVM_EXIT_RDTSC, SVM_CTRL_INTERCEPT_RDTSC);
5802	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, SVM_EXIT_RDTSC);
5803	SET_INCP_XBM_IF_EITHER_EN(INSTR_RDTSCP, SVM_EXIT_RDTSCP, SVM_CTRL_INTERCEPT_RDTSCP);
5804	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, SVM_EXIT_RDTSCP);
5805	SET_INCP_XBM_IF_EITHER_EN(INSTR_RSM, SVM_EXIT_RSM, SVM_CTRL_INTERCEPT_RSM);
5806	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, SVM_EXIT_RSM);
5807
5808	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
5809	pDbgState->bmInterceptRdCRxExtra = 0xffff;
5810	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ) \|\| IS_EITHER_ENABLED(pVM, EXIT_CRX_READ))
5811	ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_READ_CR0, SVM_EXIT_READ_CR15 + 1);
5812
5813	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
5814	pDbgState->bmInterceptWrCRxExtra = 0xffff;
5815	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE) \|\| IS_EITHER_ENABLED(pVM, EXIT_CRX_WRITE))
5816	{
5817	ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_WRITE_CR0, SVM_EXIT_WRITE_CR15 + 1);
5818	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_CR0_SEL_WRITE);
5819	}
5820
5821	if (IS_EITHER_ENABLED(pVM, INSTR_DRX_READ))
5822	pDbgState->bmInterceptRdDRxExtra = 0xffff;
5823	if (IS_EITHER_ENABLED(pVM, INSTR_DRX_READ) \|\| IS_EITHER_ENABLED(pVM, EXIT_DRX_READ))
5824	ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_READ_DR0, SVM_EXIT_READ_DR15 + 1);
5825
5826	if (IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
5827	pDbgState->bmInterceptWrDRxExtra = 0xffff;
5828	if (IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE) \|\| IS_EITHER_ENABLED(pVM, EXIT_DRX_WRITE))
5829	ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_WRITE_DR0, SVM_EXIT_WRITE_DR15 + 1);
5830
5831	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RDMSR, SVM_EXIT_MSR); /** @todo modify bitmap to intercept almost everything? (Clearing MSR_PROT just means no intercepts.) */
5832	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, SVM_EXIT_MSR);
5833	SET_ONLY_XBM_IF_EITHER_EN(INSTR_WRMSR, SVM_EXIT_MSR); /** @todo ditto */
5834	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, SVM_EXIT_MSR);
5835	SET_INCP_XBM_IF_EITHER_EN(INSTR_MWAIT, SVM_EXIT_MWAIT, SVM_CTRL_INTERCEPT_MWAIT);
5836	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, SVM_EXIT_MWAIT);
5837	if (ASMBitTest(pDbgState->bmExitsToCheck, SVM_EXIT_MWAIT))
5838	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_MWAIT_ARMED);
5839	SET_INCP_XBM_IF_EITHER_EN(INSTR_MONITOR, SVM_EXIT_MONITOR, SVM_CTRL_INTERCEPT_MONITOR);
5840	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, SVM_EXIT_MONITOR);
5841	SET_INCP_XBM_IF_EITHER_EN(INSTR_PAUSE, SVM_EXIT_PAUSE, SVM_CTRL_INTERCEPT_PAUSE);
5842	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, SVM_EXIT_PAUSE);
5843	SET_INCP_XBM_IF_EITHER_EN(INSTR_SIDT, SVM_EXIT_IDTR_READ, SVM_CTRL_INTERCEPT_IDTR_READS);
5844	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, SVM_EXIT_IDTR_READ);
5845	SET_INCP_XBM_IF_EITHER_EN(INSTR_LIDT, SVM_EXIT_IDTR_WRITE, SVM_CTRL_INTERCEPT_IDTR_WRITES);
5846	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, SVM_EXIT_IDTR_WRITE);
5847	SET_INCP_XBM_IF_EITHER_EN(INSTR_SGDT, SVM_EXIT_GDTR_READ, SVM_CTRL_INTERCEPT_GDTR_READS);
5848	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, SVM_EXIT_GDTR_READ);
5849	SET_INCP_XBM_IF_EITHER_EN(INSTR_LGDT, SVM_EXIT_GDTR_WRITE, SVM_CTRL_INTERCEPT_GDTR_WRITES);
5850	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, SVM_EXIT_GDTR_WRITE);
5851	SET_INCP_XBM_IF_EITHER_EN(INSTR_SLDT, SVM_EXIT_LDTR_READ, SVM_CTRL_INTERCEPT_LDTR_READS);
5852	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, SVM_EXIT_LDTR_READ);
5853	SET_INCP_XBM_IF_EITHER_EN(INSTR_LLDT, SVM_EXIT_LDTR_WRITE, SVM_CTRL_INTERCEPT_LDTR_WRITES);
5854	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, SVM_EXIT_LDTR_WRITE);
5855	SET_INCP_XBM_IF_EITHER_EN(INSTR_STR, SVM_EXIT_TR_READ, SVM_CTRL_INTERCEPT_TR_READS);
5856	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, SVM_EXIT_TR_READ);
5857	SET_INCP_XBM_IF_EITHER_EN(INSTR_LTR, SVM_EXIT_TR_WRITE, SVM_CTRL_INTERCEPT_TR_WRITES);
5858	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, SVM_EXIT_TR_WRITE);
5859	SET_INCP_XBM_IF_EITHER_EN(INSTR_WBINVD, SVM_EXIT_WBINVD, SVM_CTRL_INTERCEPT_WBINVD);
5860	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, SVM_EXIT_WBINVD);
5861	SET_INCP_XBM_IF_EITHER_EN(INSTR_XSETBV, SVM_EXIT_XSETBV, SVM_CTRL_INTERCEPT_XSETBV);
5862	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, SVM_EXIT_XSETBV);
5863
5864	if (DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_TRIPLE_FAULT))
5865	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_SHUTDOWN);
5866
5867	#undef IS_EITHER_ENABLED
5868	#undef SET_ONLY_XBM_IF_EITHER_EN
5869	#undef SET_INCP_XBM_IF_EITHER_EN
5870
5871	/*
5872	* Sanitize the control stuff.
5873	*/
5874	/** @todo filter out unsupported stuff? */
5875	if ( pVCpu->hmr0.s.fDebugWantRdTscExit
5876	!= RT_BOOL(pDbgState->bmInterceptExtra & (SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP)))
5877	{
5878	pVCpu->hmr0.s.fDebugWantRdTscExit ^= true;
5879	/// @todo pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
5880	RT_NOREF(pSvmTransient);
5881	}
5882
5883	Log6(("HM: debug state: bmInterceptExtra=%#RX64 bmXcptExtra=%#RX32%s%s%s%s bmExitsToCheck=%08RX32'%08RX32'%08RX32'%08RX32'%08RX32\n",
5884	pDbgState->bmInterceptExtra, pDbgState->bmXcptExtra,
5885	pDbgState->bmInterceptRdCRxExtra ? " rd-cr" : "",
5886	pDbgState->bmInterceptWrCRxExtra ? " wr-cr" : "",
5887	pDbgState->bmInterceptRdDRxExtra ? " rd-dr" : "",
5888	pDbgState->bmInterceptWrDRxExtra ? " wr-dr" : "",
5889	pDbgState->bmExitsToCheck[0],
5890	pDbgState->bmExitsToCheck[1],
5891	pDbgState->bmExitsToCheck[2],
5892	pDbgState->bmExitsToCheck[3],
5893	pDbgState->bmExitsToCheck[4]));
5894	}
5895
5896
5897	/**
5898	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
5899	* appropriate.
5900	*
5901	* The caller has checked the VM-exit against the SVMRUNDBGSTATE::bmExitsToCheck
5902	* bitmap.
5903	*
5904	* @returns Strict VBox status code (i.e. informational status codes too).
5905	* @param pVCpu The cross context virtual CPU structure.
5906	* @param pSvmTransient The SVM-transient structure.
5907	* @param uExitCode The VM-exit code.
5908	*
5909	* @remarks The name of this function is displayed by dtrace, so keep it short
5910	* and to the point. No longer than 33 chars long, please.
5911	*/
5912	static VBOXSTRICTRC hmR0SvmHandleExitDtraceEvents(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, uint64_t uExitCode)
5913	{
5914	/*
5915	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
5916	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
5917	*
5918	* Note! This is the reverse operation of what hmR0SvmPreRunGuestDebugStateUpdate
5919	* does. Must add/change/remove both places. Same ordering, please.
5920	*
5921	* Added/removed events must also be reflected in the next section
5922	* where we dispatch dtrace events.
5923	*/
5924	bool fDtrace1 = false;
5925	bool fDtrace2 = false;
5926	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
5927	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
5928	uint64_t uEventArg = 0;
5929	#define SET_XCPT(a_XcptName) \
5930	do { \
5931	enmEvent2 = RT_CONCAT(DBGFEVENT_XCPT_, a_XcptName); \
5932	fDtrace2 = RT_CONCAT3(VBOXVMM_XCPT_, a_XcptName, _ENABLED)(); \
5933	} while (0)
5934	#define SET_EXIT(a_EventSubName) \
5935	do { \
5936	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
5937	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
5938	} while (0)
5939	#define SET_BOTH(a_EventSubName) \
5940	do { \
5941	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
5942	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
5943	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
5944	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
5945	} while (0)
5946	switch (uExitCode)
5947	{
5948	case SVM_EXIT_SWINT:
5949	enmEvent2 = DBGFEVENT_INTERRUPT_SOFTWARE;
5950	fDtrace2 = VBOXVMM_INT_SOFTWARE_ENABLED();
5951	uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1;
5952	break;
5953
5954	case SVM_EXIT_XCPT_DE: SET_XCPT(DE); break;
5955	case SVM_EXIT_XCPT_DB: SET_XCPT(DB); break;
5956	case SVM_EXIT_XCPT_BP: SET_XCPT(BP); break;
5957	case SVM_EXIT_XCPT_OF: SET_XCPT(OF); break;
5958	case SVM_EXIT_XCPT_BR: SET_XCPT(BR); break;
5959	case SVM_EXIT_XCPT_UD: SET_XCPT(UD); break;
5960	case SVM_EXIT_XCPT_NM: SET_XCPT(NM); break;
5961	case SVM_EXIT_XCPT_DF: SET_XCPT(DF); break;
5962	case SVM_EXIT_XCPT_TS: SET_XCPT(TS); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5963	case SVM_EXIT_XCPT_NP: SET_XCPT(NP); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5964	case SVM_EXIT_XCPT_SS: SET_XCPT(SS); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5965	case SVM_EXIT_XCPT_GP: SET_XCPT(GP); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5966	case SVM_EXIT_XCPT_PF: SET_XCPT(PF); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5967	case SVM_EXIT_XCPT_MF: SET_XCPT(MF); break;
5968	case SVM_EXIT_XCPT_AC: SET_XCPT(AC); break;
5969	case SVM_EXIT_XCPT_XF: SET_XCPT(XF); break;
5970	case SVM_EXIT_XCPT_VE: SET_XCPT(VE); break;
5971	case SVM_EXIT_XCPT_SX: SET_XCPT(SX); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5972
5973	case SVM_EXIT_XCPT_2: enmEvent2 = DBGFEVENT_XCPT_02; break;
5974	case SVM_EXIT_XCPT_9: enmEvent2 = DBGFEVENT_XCPT_09; break;
5975	case SVM_EXIT_XCPT_15: enmEvent2 = DBGFEVENT_XCPT_0f; break;
5976	case SVM_EXIT_XCPT_18: enmEvent2 = DBGFEVENT_XCPT_MC; break;
5977	case SVM_EXIT_XCPT_21: enmEvent2 = DBGFEVENT_XCPT_15; break;
5978	case SVM_EXIT_XCPT_22: enmEvent2 = DBGFEVENT_XCPT_16; break;
5979	case SVM_EXIT_XCPT_23: enmEvent2 = DBGFEVENT_XCPT_17; break;
5980	case SVM_EXIT_XCPT_24: enmEvent2 = DBGFEVENT_XCPT_18; break;
5981	case SVM_EXIT_XCPT_25: enmEvent2 = DBGFEVENT_XCPT_19; break;
5982	case SVM_EXIT_XCPT_26: enmEvent2 = DBGFEVENT_XCPT_1a; break;
5983	case SVM_EXIT_XCPT_27: enmEvent2 = DBGFEVENT_XCPT_1b; break;
5984	case SVM_EXIT_XCPT_28: enmEvent2 = DBGFEVENT_XCPT_1c; break;
5985	case SVM_EXIT_XCPT_29: enmEvent2 = DBGFEVENT_XCPT_1d; break;
5986	case SVM_EXIT_XCPT_31: enmEvent2 = DBGFEVENT_XCPT_1f; break;
5987
5988	case SVM_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
5989	case SVM_EXIT_VMMCALL: SET_BOTH(VMM_CALL); break;
5990	case SVM_EXIT_VMRUN: SET_BOTH(SVM_VMRUN); break;
5991	case SVM_EXIT_VMLOAD: SET_BOTH(SVM_VMLOAD); break;
5992	case SVM_EXIT_VMSAVE: SET_BOTH(SVM_VMSAVE); break;
5993	case SVM_EXIT_STGI: SET_BOTH(SVM_STGI); break;
5994	case SVM_EXIT_CLGI: SET_BOTH(SVM_CLGI); break;
5995	case SVM_EXIT_CPUID: SET_BOTH(CPUID); break;
5996	case SVM_EXIT_HLT: SET_BOTH(HALT); break;
5997	case SVM_EXIT_INVD: SET_BOTH(INVD); break;
5998	case SVM_EXIT_INVLPG: SET_BOTH(INVLPG); break;
5999	case SVM_EXIT_RDPMC: SET_BOTH(RDPMC); break;
6000	case SVM_EXIT_RDTSC: SET_BOTH(RDTSC); break;
6001	case SVM_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
6002	case SVM_EXIT_RSM: SET_BOTH(RSM); break;
6003
6004	case SVM_EXIT_READ_CR0: case SVM_EXIT_READ_CR1: case SVM_EXIT_READ_CR2: case SVM_EXIT_READ_CR3:
6005	case SVM_EXIT_READ_CR4: case SVM_EXIT_READ_CR5: case SVM_EXIT_READ_CR6: case SVM_EXIT_READ_CR7:
6006	case SVM_EXIT_READ_CR8: case SVM_EXIT_READ_CR9: case SVM_EXIT_READ_CR10: case SVM_EXIT_READ_CR11:
6007	case SVM_EXIT_READ_CR12: case SVM_EXIT_READ_CR13: case SVM_EXIT_READ_CR14: case SVM_EXIT_READ_CR15:
6008	SET_BOTH(CRX_READ);
6009	uEventArg = uExitCode - SVM_EXIT_READ_CR0;
6010	break;
6011	case SVM_EXIT_WRITE_CR0: case SVM_EXIT_WRITE_CR1: case SVM_EXIT_WRITE_CR2: case SVM_EXIT_WRITE_CR3:
6012	case SVM_EXIT_WRITE_CR4: case SVM_EXIT_WRITE_CR5: case SVM_EXIT_WRITE_CR6: case SVM_EXIT_WRITE_CR7:
6013	case SVM_EXIT_WRITE_CR8: case SVM_EXIT_WRITE_CR9: case SVM_EXIT_WRITE_CR10: case SVM_EXIT_WRITE_CR11:
6014	case SVM_EXIT_WRITE_CR12: case SVM_EXIT_WRITE_CR13: case SVM_EXIT_WRITE_CR14: case SVM_EXIT_WRITE_CR15:
6015	case SVM_EXIT_CR0_SEL_WRITE:
6016	SET_BOTH(CRX_WRITE);
6017	uEventArg = uExitCode - SVM_EXIT_WRITE_CR0;
6018	break;
6019	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
6020	case SVM_EXIT_READ_DR4: case SVM_EXIT_READ_DR5: case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7:
6021	case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9: case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11:
6022	case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13: case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
6023	SET_BOTH(DRX_READ);
6024	uEventArg = uExitCode - SVM_EXIT_READ_DR0;
6025	break;
6026	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
6027	case SVM_EXIT_WRITE_DR4: case SVM_EXIT_WRITE_DR5: case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7:
6028	case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9: case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11:
6029	case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13: case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
6030	SET_BOTH(DRX_WRITE);
6031	uEventArg = uExitCode - SVM_EXIT_WRITE_DR0;
6032	break;
6033	case SVM_EXIT_MSR:
6034	if (pSvmTransient->pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_WRITE)
6035	SET_BOTH(WRMSR);
6036	else
6037	SET_BOTH(RDMSR);
6038	break;
6039	case SVM_EXIT_MWAIT_ARMED:
6040	case SVM_EXIT_MWAIT: SET_BOTH(MWAIT); break;
6041	case SVM_EXIT_MONITOR: SET_BOTH(MONITOR); break;
6042	case SVM_EXIT_PAUSE: SET_BOTH(PAUSE); break;
6043	case SVM_EXIT_IDTR_READ: SET_BOTH(SIDT); break;
6044	case SVM_EXIT_IDTR_WRITE: SET_BOTH(LIDT); break;
6045	case SVM_EXIT_GDTR_READ: SET_BOTH(SGDT); break;
6046	case SVM_EXIT_GDTR_WRITE: SET_BOTH(LGDT); break;
6047	case SVM_EXIT_LDTR_READ: SET_BOTH(SLDT); break;
6048	case SVM_EXIT_LDTR_WRITE: SET_BOTH(LLDT); break;
6049	case SVM_EXIT_TR_READ: SET_BOTH(STR); break;
6050	case SVM_EXIT_TR_WRITE: SET_BOTH(LTR); break;
6051	case SVM_EXIT_WBINVD: SET_BOTH(WBINVD); break;
6052	case SVM_EXIT_XSETBV: SET_BOTH(XSETBV); break;
6053
6054	case SVM_EXIT_SHUTDOWN:
6055	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
6056	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
6057	break;
6058
6059	default:
6060	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitCode));
6061	break;
6062	}
6063	#undef SET_BOTH
6064	#undef SET_EXIT
6065
6066	/*
6067	* Dtrace tracepoints go first. We do them here at once so we don't
6068	* have to copy the guest state saving and stuff a few dozen times.
6069	* Down side is that we've got to repeat the switch, though this time
6070	* we use enmEvent since the probes are a subset of what DBGF does.
6071	*/
6072	if (fDtrace1 \|\| fDtrace2)
6073	{
6074	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
6075	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6076	switch (enmEvent1)
6077	{
6078	/** @todo consider which extra parameters would be helpful for each probe. */
6079	case DBGFEVENT_END: break;
6080	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6081	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
6082	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
6083	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
6084	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
6085	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
6086	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
6087	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
6088	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
6089	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, (uint32_t)uEventArg); break;
6090	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, (uint32_t)uEventArg); break;
6091	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, (uint32_t)uEventArg); break;
6092	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, (uint32_t)uEventArg); break;
6093	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, (uint32_t)uEventArg, pCtx->cr2); break;
6094	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
6095	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
6096	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
6097	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
6098	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, (uint32_t)uEventArg); break;
6099	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
6100	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
6101	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
6102	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
6103	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
6104	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
6105	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
6106	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6107	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6108	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6109	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6110	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
6111	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
6112	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
6113	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
6114	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
6115	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
6116	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
6117	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
6118	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
6119	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
6120	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
6121	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
6122	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
6123	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
6124	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
6125	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
6126	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
6127	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
6128	case DBGFEVENT_INSTR_SVM_VMRUN: VBOXVMM_INSTR_SVM_VMRUN(pVCpu, pCtx); break;
6129	case DBGFEVENT_INSTR_SVM_VMLOAD: VBOXVMM_INSTR_SVM_VMLOAD(pVCpu, pCtx); break;
6130	case DBGFEVENT_INSTR_SVM_VMSAVE: VBOXVMM_INSTR_SVM_VMSAVE(pVCpu, pCtx); break;
6131	case DBGFEVENT_INSTR_SVM_STGI: VBOXVMM_INSTR_SVM_STGI(pVCpu, pCtx); break;
6132	case DBGFEVENT_INSTR_SVM_CLGI: VBOXVMM_INSTR_SVM_CLGI(pVCpu, pCtx); break;
6133	default: AssertMsgFailed(("enmEvent1=%d uExitCode=%d\n", enmEvent1, uExitCode)); break;
6134	}
6135	switch (enmEvent2)
6136	{
6137	/** @todo consider which extra parameters would be helpful for each probe. */
6138	case DBGFEVENT_END: break;
6139	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
6140	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
6141	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
6142	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
6143	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
6144	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
6145	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
6146	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
6147	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6148	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6149	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6150	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6151	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
6152	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
6153	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
6154	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
6155	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
6156	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
6157	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
6158	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
6159	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
6160	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
6161	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
6162	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
6163	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
6164	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
6165	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
6166	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
6167	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
6168	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
6169	case DBGFEVENT_EXIT_SVM_VMRUN: VBOXVMM_EXIT_SVM_VMRUN(pVCpu, pCtx); break;
6170	case DBGFEVENT_EXIT_SVM_VMLOAD: VBOXVMM_EXIT_SVM_VMLOAD(pVCpu, pCtx); break;
6171	case DBGFEVENT_EXIT_SVM_VMSAVE: VBOXVMM_EXIT_SVM_VMSAVE(pVCpu, pCtx); break;
6172	case DBGFEVENT_EXIT_SVM_STGI: VBOXVMM_EXIT_SVM_STGI(pVCpu, pCtx); break;
6173	case DBGFEVENT_EXIT_SVM_CLGI: VBOXVMM_EXIT_SVM_CLGI(pVCpu, pCtx); break;
6174	default: AssertMsgFailed(("enmEvent2=%d uExitCode=%d\n", enmEvent2, uExitCode)); break;
6175	}
6176	}
6177
6178	/*
6179	* Fire of the DBGF event, if enabled (our check here is just a quick one,
6180	* the DBGF call will do a full check).
6181	*
6182	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
6183	* Note! If we have to events, we prioritize the first, i.e. the instruction
6184	* one, in order to avoid event nesting.
6185	*/
6186	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6187	VBOXSTRICTRC rcStrict;
6188	if ( enmEvent1 != DBGFEVENT_END
6189	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
6190	{
6191	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6192	rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
6193	}
6194	else if ( enmEvent2 != DBGFEVENT_END
6195	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
6196	{
6197	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6198	rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
6199	}
6200	else
6201	rcStrict = VINF_SUCCESS;
6202	return rcStrict;
6203	}
6204
6205
6206	/**
6207	* Handles a guest \#VMEXIT (for all EXITCODE values except SVM_EXIT_INVALID),
6208	* debug variant.
6209	*
6210	* @returns Strict VBox status code (informational status codes included).
6211	* @param pVCpu The cross context virtual CPU structure.
6212	* @param pSvmTransient Pointer to the SVM transient structure.
6213	* @param pDbgState The runtime debug state.
6214	*/
6215	static VBOXSTRICTRC hmR0SvmDebugHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
6216	{
6217	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
6218	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
6219
6220	/*
6221	* Expensive (saves context) generic dtrace VM-exit probe.
6222	*/
6223	uint64_t const uExitCode = pSvmTransient->u64ExitCode;
6224	if (!VBOXVMM_R0_HMSVM_VMEXIT_ENABLED())
6225	{ /* more likely */ }
6226	else
6227	{
6228	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
6229	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, uExitCode, pSvmTransient->pVmcb);
6230	}
6231
6232	/*
6233	* Check for single stepping event if we're stepping.
6234	*/
6235	if (pVCpu->hm.s.fSingleInstruction)
6236	{
6237	switch (uExitCode)
6238	{
6239	/* Various events: */
6240	case SVM_EXIT_XCPT_0: case SVM_EXIT_XCPT_1: case SVM_EXIT_XCPT_2: case SVM_EXIT_XCPT_3:
6241	case SVM_EXIT_XCPT_4: case SVM_EXIT_XCPT_5: case SVM_EXIT_XCPT_6: case SVM_EXIT_XCPT_7:
6242	case SVM_EXIT_XCPT_8: case SVM_EXIT_XCPT_9: case SVM_EXIT_XCPT_10: case SVM_EXIT_XCPT_11:
6243	case SVM_EXIT_XCPT_12: case SVM_EXIT_XCPT_13: case SVM_EXIT_XCPT_14: case SVM_EXIT_XCPT_15:
6244	case SVM_EXIT_XCPT_16: case SVM_EXIT_XCPT_17: case SVM_EXIT_XCPT_18: case SVM_EXIT_XCPT_19:
6245	case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
6246	case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
6247	case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
6248	case SVM_EXIT_INTR:
6249	case SVM_EXIT_NMI:
6250	case SVM_EXIT_VINTR:
6251	case SVM_EXIT_NPF:
6252	case SVM_EXIT_AVIC_NOACCEL:
6253
6254	/* Instruction specific VM-exits: */
6255	case SVM_EXIT_READ_CR0: case SVM_EXIT_READ_CR1: case SVM_EXIT_READ_CR2: case SVM_EXIT_READ_CR3:
6256	case SVM_EXIT_READ_CR4: case SVM_EXIT_READ_CR5: case SVM_EXIT_READ_CR6: case SVM_EXIT_READ_CR7:
6257	case SVM_EXIT_READ_CR8: case SVM_EXIT_READ_CR9: case SVM_EXIT_READ_CR10: case SVM_EXIT_READ_CR11:
6258	case SVM_EXIT_READ_CR12: case SVM_EXIT_READ_CR13: case SVM_EXIT_READ_CR14: case SVM_EXIT_READ_CR15:
6259	case SVM_EXIT_WRITE_CR0: case SVM_EXIT_WRITE_CR1: case SVM_EXIT_WRITE_CR2: case SVM_EXIT_WRITE_CR3:
6260	case SVM_EXIT_WRITE_CR4: case SVM_EXIT_WRITE_CR5: case SVM_EXIT_WRITE_CR6: case SVM_EXIT_WRITE_CR7:
6261	case SVM_EXIT_WRITE_CR8: case SVM_EXIT_WRITE_CR9: case SVM_EXIT_WRITE_CR10: case SVM_EXIT_WRITE_CR11:
6262	case SVM_EXIT_WRITE_CR12: case SVM_EXIT_WRITE_CR13: case SVM_EXIT_WRITE_CR14: case SVM_EXIT_WRITE_CR15:
6263	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
6264	case SVM_EXIT_READ_DR4: case SVM_EXIT_READ_DR5: case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7:
6265	case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9: case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11:
6266	case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13: case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
6267	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
6268	case SVM_EXIT_WRITE_DR4: case SVM_EXIT_WRITE_DR5: case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7:
6269	case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9: case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11:
6270	case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13: case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
6271	case SVM_EXIT_CR0_SEL_WRITE:
6272	case SVM_EXIT_IDTR_READ:
6273	case SVM_EXIT_GDTR_READ:
6274	case SVM_EXIT_LDTR_READ:
6275	case SVM_EXIT_TR_READ:
6276	case SVM_EXIT_IDTR_WRITE:
6277	case SVM_EXIT_GDTR_WRITE:
6278	case SVM_EXIT_LDTR_WRITE:
6279	case SVM_EXIT_TR_WRITE:
6280	case SVM_EXIT_RDTSC:
6281	case SVM_EXIT_RDPMC:
6282	case SVM_EXIT_PUSHF:
6283	case SVM_EXIT_POPF:
6284	case SVM_EXIT_CPUID:
6285	case SVM_EXIT_RSM:
6286	case SVM_EXIT_IRET:
6287	case SVM_EXIT_SWINT:
6288	case SVM_EXIT_INVD:
6289	case SVM_EXIT_PAUSE:
6290	case SVM_EXIT_HLT:
6291	case SVM_EXIT_INVLPG:
6292	case SVM_EXIT_INVLPGA:
6293	case SVM_EXIT_IOIO:
6294	case SVM_EXIT_MSR:
6295	case SVM_EXIT_TASK_SWITCH:
6296	case SVM_EXIT_VMRUN:
6297	case SVM_EXIT_VMMCALL:
6298	case SVM_EXIT_VMLOAD:
6299	case SVM_EXIT_VMSAVE:
6300	case SVM_EXIT_STGI:
6301	case SVM_EXIT_CLGI:
6302	case SVM_EXIT_SKINIT:
6303	case SVM_EXIT_RDTSCP:
6304	case SVM_EXIT_ICEBP:
6305	case SVM_EXIT_WBINVD:
6306	case SVM_EXIT_MONITOR:
6307	case SVM_EXIT_MWAIT:
6308	case SVM_EXIT_MWAIT_ARMED:
6309	case SVM_EXIT_XSETBV:
6310	case SVM_EXIT_RDPRU:
6311	case SVM_EXIT_WRITE_EFER_TRAP:
6312	case SVM_EXIT_WRITE_CR0_TRAP: case SVM_EXIT_WRITE_CR1_TRAP: case SVM_EXIT_WRITE_CR2_TRAP: case SVM_EXIT_WRITE_CR3_TRAP:
6313	case SVM_EXIT_WRITE_CR4_TRAP: case SVM_EXIT_WRITE_CR5_TRAP: case SVM_EXIT_WRITE_CR6_TRAP: case SVM_EXIT_WRITE_CR7_TRAP:
6314	case SVM_EXIT_WRITE_CR8_TRAP: case SVM_EXIT_WRITE_CR9_TRAP: case SVM_EXIT_WRITE_CR10_TRAP: case SVM_EXIT_WRITE_CR11_TRAP:
6315	case SVM_EXIT_WRITE_CR12_TRAP: case SVM_EXIT_WRITE_CR13_TRAP: case SVM_EXIT_WRITE_CR14_TRAP: case SVM_EXIT_WRITE_CR15_TRAP:
6316	case SVM_EXIT_MCOMMIT:
6317	{
6318	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6319	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
6320	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
6321	{
6322	Log6Func(("VINF_EM_DBG_STEPPED: %04x:%08RX64 (exit %u)\n",
6323	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uExitCode));
6324	return VINF_EM_DBG_STEPPED;
6325	}
6326	break;
6327	}
6328
6329	/* Errors and unexpected events: */
6330	case SVM_EXIT_FERR_FREEZE:
6331	case SVM_EXIT_SHUTDOWN:
6332	case SVM_EXIT_AVIC_INCOMPLETE_IPI:
6333	break;
6334
6335	case SVM_EXIT_SMI:
6336	case SVM_EXIT_INIT:
6337	default:
6338	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitCode));
6339	break;
6340	}
6341	}
6342
6343	/*
6344	* Check for debugger event breakpoints and dtrace probes.
6345	*/
6346	if ( uExitCode < sizeof(pDbgState->bmExitsToCheck) * 8U
6347	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitCode) )
6348	{
6349	VBOXSTRICTRC rcStrict = hmR0SvmHandleExitDtraceEvents(pVCpu, pSvmTransient, uExitCode);
6350	if (rcStrict != VINF_SUCCESS)
6351	{
6352	Log6Func(("%04x:%08RX64 (exit %u) -> %Rrc\n",
6353	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uExitCode, VBOXSTRICTRC_VAL(rcStrict) ));
6354	return rcStrict;
6355	}
6356	}
6357
6358	/*
6359	* Normal processing.
6360	*/
6361	return hmR0SvmHandleExit(pVCpu, pSvmTransient);
6362	}
6363
6364
6365	/**
6366	* Runs the guest code using AMD-V in single step mode.
6367	*
6368	* @returns Strict VBox status code.
6369	* @param pVCpu The cross context virtual CPU structure.
6370	* @param pcLoops Pointer to the number of executed loops.
6371	*/
6372	static VBOXSTRICTRC hmR0SvmRunGuestCodeDebug(PVMCPUCC pVCpu, uint32_t *pcLoops)
6373	{
6374	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops;
6375	Assert(pcLoops);
6376	Assert(*pcLoops <= cMaxResumeLoops);
6377
6378	SVMTRANSIENT SvmTransient;
6379	RT_ZERO(SvmTransient);
6380	SvmTransient.fUpdateTscOffsetting = true;
6381	SvmTransient.pVmcb = pVCpu->hmr0.s.svm.pVmcb;
6382
6383	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6384
6385	/* Set HMCPU indicators. */
6386	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
6387	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
6388	pVCpu->hmr0.s.fDebugWantRdTscExit = false;
6389	pVCpu->hmr0.s.fUsingDebugLoop = true;
6390
6391	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
6392	SVMRUNDBGSTATE DbgState;
6393	hmR0SvmRunDebugStateInit(pVCpu, &SvmTransient, &DbgState);
6394	hmR0SvmPreRunGuestDebugStateUpdate(pVCpu, &SvmTransient, &DbgState);
6395
6396	/*
6397	* The loop.
6398	*/
6399	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_5;
6400	for (;;)
6401	{
6402	Assert(!HMR0SuspendPending());
6403	AssertMsg(pVCpu->hmr0.s.idEnteredCpu == RTMpCpuId(),
6404	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hmr0.s.idEnteredCpu,
6405	(unsigned)RTMpCpuId(), *pcLoops));
6406	bool fStepping = pVCpu->hm.s.fSingleInstruction;
6407
6408	/* Set up VM-execution controls the next two can respond to. */
6409	hmR0SvmPreRunGuestDebugStateApply(&SvmTransient, &DbgState);
6410
6411	/* Preparatory work for running nested-guest code, this may force us to return to
6412	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
6413	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6414	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
6415	if (rc != VINF_SUCCESS)
6416	break;
6417
6418	/*
6419	* No longjmps to ring-3 from this point on!!!
6420	*
6421	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
6422	* better than a kernel panic. This also disables flushing of the R0-logger instance.
6423	*/
6424	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
6425
6426	/* Override any obnoxious code in the above two calls. */
6427	hmR0SvmPreRunGuestDebugStateApply(&SvmTransient, &DbgState);
6428	#if 0
6429	Log(("%04x:%08RX64 ds=%04x %04x:%08RX64 i=%#RX64\n",
6430	SvmTransient.pVmcb->guest.CS.u16Sel, SvmTransient.pVmcb->guest.u64RIP, SvmTransient.pVmcb->guest.DS.u16Sel,
6431	SvmTransient.pVmcb->guest.SS.u16Sel, SvmTransient.pVmcb->guest.u64RSP, SvmTransient.pVmcb->ctrl.EventInject.u));
6432	#endif
6433
6434	/*
6435	* Finally execute guest code.
6436	*/
6437	rc = hmR0SvmRunGuest(pVCpu, pVCpu->hmr0.s.svm.HCPhysVmcb);
6438
6439	/* Restore any residual host-state and save any bits shared between host and guest
6440	into the guest-CPU state. Re-enables interrupts! */
6441	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
6442	#if 0
6443	Log(("%04x:%08RX64 ds=%04x %04x:%08RX64 i=%#RX64 exit=%d\n",
6444	SvmTransient.pVmcb->guest.CS.u16Sel, SvmTransient.pVmcb->guest.u64RIP, SvmTransient.pVmcb->guest.DS.u16Sel,
6445	SvmTransient.pVmcb->guest.SS.u16Sel, SvmTransient.pVmcb->guest.u64RSP, SvmTransient.pVmcb->ctrl.EventInject.u, SvmTransient.u64ExitCode));
6446	#endif
6447
6448	if (RT_LIKELY( rc == VINF_SUCCESS /* Check for VMRUN errors. */
6449	&& SvmTransient.u64ExitCode != SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
6450	{ /* very likely*/ }
6451	else
6452	{
6453	if (rc == VINF_SUCCESS)
6454	rc = VERR_SVM_INVALID_GUEST_STATE;
6455	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
6456	hmR0SvmReportWorldSwitchError(pVCpu, VBOXSTRICTRC_VAL(rc));
6457	return rc;
6458	}
6459
6460	/* Handle the #VMEXIT. */
6461	HMSVM_DEBUG_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
6462	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
6463	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, pVCpu->hmr0.s.svm.pVmcb);
6464	rc = hmR0SvmDebugHandleExit(pVCpu, &SvmTransient, &DbgState);
6465	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
6466	if (rc != VINF_SUCCESS)
6467	break;
6468	if (++(*pcLoops) >= cMaxResumeLoops)
6469	{
6470	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
6471	rc = VINF_EM_RAW_INTERRUPT;
6472	break;
6473	}
6474
6475	/*
6476	* Stepping: Did the RIP change, if so, consider it a single step.
6477	* Otherwise, make sure one of the TFs gets set.
6478	*/
6479	if (fStepping)
6480	{
6481	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6482	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
6483	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
6484	{
6485	Log6Func(("VINF_EM_DBG_STEPPED: %04x:%08RX64 (exit %u)\n",
6486	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, SvmTransient.u64ExitCode));
6487	rc = VINF_EM_DBG_STEPPED;
6488	break;
6489	}
6490	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
6491	}
6492
6493	/*
6494	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
6495	* Revert the state changes afterware so we can drop intercepts no longer needed.
6496	*/
6497	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
6498	{
6499	hmR0SvmPreRunGuestDebugStateUpdate(pVCpu, &SvmTransient, &DbgState);
6500	hmR0SvmRunDebugStateRevert(&SvmTransient, &DbgState);
6501	}
6502	}
6503
6504	/*
6505	* Clear the X86_EFL_TF if necessary.
6506	*/
6507	if (pVCpu->hmr0.s.fClearTrapFlag)
6508	{
6509	pVCpu->hmr0.s.fClearTrapFlag = false;
6510	pCtx->eflags.Bits.u1TF = 0;
6511	}
6512
6513	/* Restore HMCPU indicators. */
6514	pVCpu->hmr0.s.fUsingDebugLoop = false;
6515	pVCpu->hmr0.s.fDebugWantRdTscExit = false;
6516	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
6517
6518	/* Restore all controls applied by hmR0SvmPreRunGuestDebugStateApply above. */
6519	hmR0SvmRunDebugStateRevert(&SvmTransient, &DbgState);
6520
6521	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6522	return rc;
6523	}
6524
6525	/** @} */
6526
6527	#undef VMEXIT_CALL_RET
6528
6529
6530	#ifdef VBOX_STRICT
6531	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6532	# define HMSVM_ASSERT_PREEMPT_CPUID_VAR() \
6533	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6534
6535	# define HMSVM_ASSERT_PREEMPT_CPUID() \
6536	do \
6537	{ \
6538	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6539	AssertMsg(idAssertCpu == idAssertCpuNow, ("SVM %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6540	} while (0)
6541
6542	# define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pSvmTransient) \
6543	do { \
6544	AssertPtr((a_pVCpu)); \
6545	AssertPtr((a_pSvmTransient)); \
6546	Assert(ASMIntAreEnabled()); \
6547	HMSVM_ASSERT_PREEMPT_SAFE((a_pVCpu)); \
6548	HMSVM_ASSERT_PREEMPT_CPUID_VAR(); \
6549	Log4Func(("vcpu[%u] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (a_pVCpu)->idCpu)); \
6550	HMSVM_ASSERT_PREEMPT_SAFE((a_pVCpu)); \
6551	if (!VMMRZCallRing3IsEnabled((a_pVCpu))) \
6552	HMSVM_ASSERT_PREEMPT_CPUID(); \
6553	} while (0)
6554	#else
6555	# define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pSvmTransient) \
6556	do { \
6557	RT_NOREF2(a_pVCpu, a_pSvmTransient); \
6558	} while (0)
6559	#endif
6560
6561
6562	/**
6563	* Gets the IEM exception flags for the specified SVM event.
6564	*
6565	* @returns The IEM exception flags.
6566	* @param pEvent Pointer to the SVM event.
6567	*
6568	* @remarks This function currently only constructs flags required for
6569	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g. error-code
6570	* and CR2 aspects of an exception are not included).
6571	*/
6572	static uint32_t hmR0SvmGetIemXcptFlags(PCSVMEVENT pEvent)
6573	{
6574	uint8_t const uEventType = pEvent->n.u3Type;
6575	uint32_t fIemXcptFlags;
6576	switch (uEventType)
6577	{
6578	case SVM_EVENT_EXCEPTION:
6579	/*
6580	* Only INT3 and INTO instructions can raise #BP and #OF exceptions.
6581	* See AMD spec. Table 8-1. "Interrupt Vector Source and Cause".
6582	*/
6583	if (pEvent->n.u8Vector == X86_XCPT_BP)
6584	{
6585	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_BP_INSTR;
6586	break;
6587	}
6588	if (pEvent->n.u8Vector == X86_XCPT_OF)
6589	{
6590	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_OF_INSTR;
6591	break;
6592	}
6593	/** @todo How do we distinguish ICEBP \#DB from the regular one? */
6594	RT_FALL_THRU();
6595	case SVM_EVENT_NMI:
6596	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6597	break;
6598
6599	case SVM_EVENT_EXTERNAL_IRQ:
6600	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6601	break;
6602
6603	case SVM_EVENT_SOFTWARE_INT:
6604	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6605	break;
6606
6607	default:
6608	fIemXcptFlags = 0;
6609	AssertMsgFailed(("Unexpected event type! uEventType=%#x uVector=%#x", uEventType, pEvent->n.u8Vector));
6610	break;
6611	}
6612	return fIemXcptFlags;
6613	}
6614
6615
6616	/**
6617	* Handle a condition that occurred while delivering an event through the guest
6618	* IDT.
6619	*
6620	* @returns VBox status code (informational error codes included).
6621	* @retval VINF_SUCCESS if we should continue handling the \#VMEXIT.
6622	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought to
6623	* continue execution of the guest which will delivery the \#DF.
6624	* @retval VINF_EM_RESET if we detected a triple-fault condition.
6625	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
6626	*
6627	* @param pVCpu The cross context virtual CPU structure.
6628	* @param pSvmTransient Pointer to the SVM transient structure.
6629	*
6630	* @remarks No-long-jump zone!!!
6631	*/
6632	static int hmR0SvmCheckExitDueToEventDelivery(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6633	{
6634	/** @todo r=bird: Looks like this is called on many exits and we start by
6635	* loading CR2 on the offchance that we actually have work to do here.
6636	*
6637	* HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY can surely check
6638	* pVmcb->ctrl.ExitIntInfo.n.u1Valid, can't it?
6639	*
6640	* Also, what's the deal with hmR0SvmGetCurrentVmcb() vs pSvmTransient->pVmcb?
6641	*/
6642	int rc = VINF_SUCCESS;
6643	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6644	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR2);
6645
6646	Log4(("EXITINTINFO: Pending vectoring event %#RX64 Valid=%RTbool ErrValid=%RTbool Err=%#RX32 Type=%u Vector=%u\n",
6647	pVmcb->ctrl.ExitIntInfo.u, !!pVmcb->ctrl.ExitIntInfo.n.u1Valid, !!pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid,
6648	pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode, pVmcb->ctrl.ExitIntInfo.n.u3Type, pVmcb->ctrl.ExitIntInfo.n.u8Vector));
6649
6650	/*
6651	* The EXITINTINFO (if valid) contains the prior exception (IDT vector) that was trying to
6652	* be delivered to the guest which caused a #VMEXIT which was intercepted (Exit vector).
6653	*
6654	* See AMD spec. 15.7.3 "EXITINFO Pseudo-Code".
6655	*/
6656	if (pVmcb->ctrl.ExitIntInfo.n.u1Valid)
6657	{
6658	IEMXCPTRAISE enmRaise;
6659	IEMXCPTRAISEINFO fRaiseInfo;
6660	bool const fExitIsHwXcpt = pSvmTransient->u64ExitCode - SVM_EXIT_XCPT_0 <= SVM_EXIT_XCPT_31;
6661	uint8_t const uIdtVector = pVmcb->ctrl.ExitIntInfo.n.u8Vector;
6662	if (fExitIsHwXcpt)
6663	{
6664	uint8_t const uExitVector = pSvmTransient->u64ExitCode - SVM_EXIT_XCPT_0;
6665	uint32_t const fIdtVectorFlags = hmR0SvmGetIemXcptFlags(&pVmcb->ctrl.ExitIntInfo);
6666	uint32_t const fExitVectorFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6667	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
6668	}
6669	else
6670	{
6671	/*
6672	* If delivery of an event caused a #VMEXIT that is not an exception (e.g. #NPF)
6673	* then we end up here.
6674	*
6675	* If the event was:
6676	* - a software interrupt, we can re-execute the instruction which will
6677	* regenerate the event.
6678	* - an NMI, we need to clear NMI blocking and re-inject the NMI.
6679	* - a hardware exception or external interrupt, we re-inject it.
6680	*/
6681	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6682	if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_SOFTWARE_INT)
6683	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
6684	else
6685	enmRaise = IEMXCPTRAISE_PREV_EVENT;
6686	}
6687
6688	switch (enmRaise)
6689	{
6690	case IEMXCPTRAISE_CURRENT_XCPT:
6691	case IEMXCPTRAISE_PREV_EVENT:
6692	{
6693	/* For software interrupts, we shall re-execute the instruction. */
6694	if (!(fRaiseInfo & IEMXCPTRAISEINFO_SOFT_INT_XCPT))
6695	{
6696	RTGCUINTPTR GCPtrFaultAddress = 0;
6697
6698	/* If we are re-injecting an NMI, clear NMI blocking. */
6699	if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_NMI)
6700	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6701
6702	/* Determine a vectoring #PF condition, see comment in hmR0SvmExitXcptPF(). */
6703	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
6704	{
6705	pSvmTransient->fVectoringPF = true;
6706	Log4Func(("IDT: Pending vectoring #PF due to delivery of Ext-Int/NMI. uCR2=%#RX64\n",
6707	pVCpu->cpum.GstCtx.cr2));
6708	}
6709	else if ( pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_EXCEPTION
6710	&& uIdtVector == X86_XCPT_PF)
6711	{
6712	/*
6713	* If the previous exception was a #PF, we need to recover the CR2 value.
6714	* This can't happen with shadow paging.
6715	*/
6716	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
6717	}
6718
6719	/*
6720	* Without nested paging, when uExitVector is #PF, CR2 value will be updated from the VMCB's
6721	* exit info. fields, if it's a guest #PF, see hmR0SvmExitXcptPF().
6722	*/
6723	Assert(pVmcb->ctrl.ExitIntInfo.n.u3Type != SVM_EVENT_SOFTWARE_INT);
6724	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflect);
6725	hmR0SvmSetPendingEvent(pVCpu, &pVmcb->ctrl.ExitIntInfo, GCPtrFaultAddress);
6726
6727	Log4Func(("IDT: Pending vectoring event %#RX64 ErrValid=%RTbool Err=%#RX32 GCPtrFaultAddress=%#RX64\n",
6728	pVmcb->ctrl.ExitIntInfo.u, RT_BOOL(pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid),
6729	pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode, GCPtrFaultAddress));
6730	}
6731	break;
6732	}
6733
6734	case IEMXCPTRAISE_REEXEC_INSTR:
6735	{
6736	Assert(rc == VINF_SUCCESS);
6737	break;
6738	}
6739
6740	case IEMXCPTRAISE_DOUBLE_FAULT:
6741	{
6742	/*
6743	* Determing a vectoring double #PF condition. Used later, when PGM evaluates
6744	* the second #PF as a guest #PF (and not a shadow #PF) and needs to be
6745	* converted into a #DF.
6746	*/
6747	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6748	{
6749	Log4Func(("IDT: Pending vectoring double #PF uCR2=%#RX64\n", pVCpu->cpum.GstCtx.cr2));
6750	pSvmTransient->fVectoringDoublePF = true;
6751	Assert(rc == VINF_SUCCESS);
6752	}
6753	else
6754	{
6755	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectConvertDF);
6756	hmR0SvmSetPendingXcptDF(pVCpu);
6757	rc = VINF_HM_DOUBLE_FAULT;
6758	}
6759	break;
6760	}
6761
6762	case IEMXCPTRAISE_TRIPLE_FAULT:
6763	{
6764	rc = VINF_EM_RESET;
6765	break;
6766	}
6767
6768	case IEMXCPTRAISE_CPU_HANG:
6769	{
6770	rc = VERR_EM_GUEST_CPU_HANG;
6771	break;
6772	}
6773
6774	default:
6775	AssertMsgFailedBreakStmt(("Bogus enmRaise value: %d (%#x)\n", enmRaise, enmRaise), rc = VERR_SVM_IPE_2);
6776	}
6777	}
6778	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET \|\| rc == VERR_EM_GUEST_CPU_HANG);
6779	return rc;
6780	}
6781
6782
6783	/**
6784	* Advances the guest RIP by the number of bytes specified in @a cb.
6785	*
6786	* @param pVCpu The cross context virtual CPU structure.
6787	* @param cb RIP increment value in bytes.
6788	*/
6789	DECLINLINE(void) hmR0SvmAdvanceRip(PVMCPUCC pVCpu, uint32_t cb)
6790	{
6791	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6792	pCtx->rip += cb;
6793
6794	/* Update interrupt shadow. */
6795	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
6796	&& pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
6797	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6798	}
6799
6800
6801	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6802	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- #VMEXIT handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6803	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6804
6805	/** @name \#VMEXIT handlers.
6806	* @{
6807	*/
6808
6809	/**
6810	* \#VMEXIT handler for external interrupts, NMIs, FPU assertion freeze and INIT
6811	* signals (SVM_EXIT_INTR, SVM_EXIT_NMI, SVM_EXIT_FERR_FREEZE, SVM_EXIT_INIT).
6812	*/
6813	HMSVM_EXIT_DECL hmR0SvmExitIntr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6814	{
6815	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6816
6817	if (pSvmTransient->u64ExitCode == SVM_EXIT_NMI)
6818	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
6819	else if (pSvmTransient->u64ExitCode == SVM_EXIT_INTR)
6820	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6821
6822	/*
6823	* AMD-V has no preemption timer and the generic periodic preemption timer has no way to
6824	* signal -before- the timer fires if the current interrupt is our own timer or a some
6825	* other host interrupt. We also cannot examine what interrupt it is until the host
6826	* actually take the interrupt.
6827	*
6828	* Going back to executing guest code here unconditionally causes random scheduling
6829	* problems (observed on an AMD Phenom 9850 Quad-Core on Windows 64-bit host).
6830	*/
6831	return VINF_EM_RAW_INTERRUPT;
6832	}
6833
6834
6835	/**
6836	* \#VMEXIT handler for WBINVD (SVM_EXIT_WBINVD). Conditional \#VMEXIT.
6837	*/
6838	HMSVM_EXIT_DECL hmR0SvmExitWbinvd(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6839	{
6840	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6841
6842	VBOXSTRICTRC rcStrict;
6843	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6844	if (fSupportsNextRipSave)
6845	{
6846	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
6847	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6848	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6849	rcStrict = IEMExecDecodedWbinvd(pVCpu, cbInstr);
6850	}
6851	else
6852	{
6853	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6854	rcStrict = IEMExecOne(pVCpu);
6855	}
6856
6857	if (rcStrict == VINF_IEM_RAISED_XCPT)
6858	{
6859	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6860	rcStrict = VINF_SUCCESS;
6861	}
6862	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6863	return rcStrict;
6864	}
6865
6866
6867	/**
6868	* \#VMEXIT handler for INVD (SVM_EXIT_INVD). Unconditional \#VMEXIT.
6869	*/
6870	HMSVM_EXIT_DECL hmR0SvmExitInvd(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6871	{
6872	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6873
6874	VBOXSTRICTRC rcStrict;
6875	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6876	if (fSupportsNextRipSave)
6877	{
6878	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
6879	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6880	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6881	rcStrict = IEMExecDecodedInvd(pVCpu, cbInstr);
6882	}
6883	else
6884	{
6885	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6886	rcStrict = IEMExecOne(pVCpu);
6887	}
6888
6889	if (rcStrict == VINF_IEM_RAISED_XCPT)
6890	{
6891	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6892	rcStrict = VINF_SUCCESS;
6893	}
6894	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6895	return rcStrict;
6896	}
6897
6898
6899	/**
6900	* \#VMEXIT handler for INVD (SVM_EXIT_CPUID). Conditional \#VMEXIT.
6901	*/
6902	HMSVM_EXIT_DECL hmR0SvmExitCpuid(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6903	{
6904	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6905
6906	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX);
6907	VBOXSTRICTRC rcStrict;
6908	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
6909	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
6910	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
6911	if (!pExitRec)
6912	{
6913	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6914	if (fSupportsNextRipSave)
6915	{
6916	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6917	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6918	rcStrict = IEMExecDecodedCpuid(pVCpu, cbInstr);
6919	}
6920	else
6921	{
6922	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6923	rcStrict = IEMExecOne(pVCpu);
6924	}
6925
6926	if (rcStrict == VINF_IEM_RAISED_XCPT)
6927	{
6928	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6929	rcStrict = VINF_SUCCESS;
6930	}
6931	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6932	}
6933	else
6934	{
6935	/*
6936	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
6937	*/
6938	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6939
6940	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
6941	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
6942
6943	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
6944
6945	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
6946	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
6947	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
6948	}
6949	return rcStrict;
6950	}
6951
6952
6953	/**
6954	* \#VMEXIT handler for RDTSC (SVM_EXIT_RDTSC). Conditional \#VMEXIT.
6955	*/
6956	HMSVM_EXIT_DECL hmR0SvmExitRdtsc(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6957	{
6958	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6959
6960	VBOXSTRICTRC rcStrict;
6961	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6962	if (fSupportsNextRipSave)
6963	{
6964	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4);
6965	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6966	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6967	rcStrict = IEMExecDecodedRdtsc(pVCpu, cbInstr);
6968	}
6969	else
6970	{
6971	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6972	rcStrict = IEMExecOne(pVCpu);
6973	}
6974
6975	if (rcStrict == VINF_SUCCESS)
6976	pSvmTransient->fUpdateTscOffsetting = true;
6977	else if (rcStrict == VINF_IEM_RAISED_XCPT)
6978	{
6979	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6980	rcStrict = VINF_SUCCESS;
6981	}
6982	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6983	return rcStrict;
6984	}
6985
6986
6987	/**
6988	* \#VMEXIT handler for RDTSCP (SVM_EXIT_RDTSCP). Conditional \#VMEXIT.
6989	*/
6990	HMSVM_EXIT_DECL hmR0SvmExitRdtscp(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6991	{
6992	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6993
6994	VBOXSTRICTRC rcStrict;
6995	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6996	if (fSupportsNextRipSave)
6997	{
6998	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_TSC_AUX);
6999	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7000	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7001	rcStrict = IEMExecDecodedRdtscp(pVCpu, cbInstr);
7002	}
7003	else
7004	{
7005	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7006	rcStrict = IEMExecOne(pVCpu);
7007	}
7008
7009	if (rcStrict == VINF_SUCCESS)
7010	pSvmTransient->fUpdateTscOffsetting = true;
7011	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7012	{
7013	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7014	rcStrict = VINF_SUCCESS;
7015	}
7016	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7017	return rcStrict;
7018	}
7019
7020
7021	/**
7022	* \#VMEXIT handler for RDPMC (SVM_EXIT_RDPMC). Conditional \#VMEXIT.
7023	*/
7024	HMSVM_EXIT_DECL hmR0SvmExitRdpmc(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7025	{
7026	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7027
7028	VBOXSTRICTRC rcStrict;
7029	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7030	if (fSupportsNextRipSave)
7031	{
7032	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4);
7033	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7034	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7035	rcStrict = IEMExecDecodedRdpmc(pVCpu, cbInstr);
7036	}
7037	else
7038	{
7039	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7040	rcStrict = IEMExecOne(pVCpu);
7041	}
7042
7043	if (rcStrict == VINF_IEM_RAISED_XCPT)
7044	{
7045	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7046	rcStrict = VINF_SUCCESS;
7047	}
7048	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7049	return rcStrict;
7050	}
7051
7052
7053	/**
7054	* \#VMEXIT handler for INVLPG (SVM_EXIT_INVLPG). Conditional \#VMEXIT.
7055	*/
7056	HMSVM_EXIT_DECL hmR0SvmExitInvlpg(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7057	{
7058	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7059	Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
7060
7061	VBOXSTRICTRC rcStrict;
7062	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
7063	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7064	if ( fSupportsDecodeAssists
7065	&& fSupportsNextRipSave)
7066	{
7067	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
7068	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7069	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7070	RTGCPTR const GCPtrPage = pVmcb->ctrl.u64ExitInfo1;
7071	rcStrict = IEMExecDecodedInvlpg(pVCpu, cbInstr, GCPtrPage);
7072	}
7073	else
7074	{
7075	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7076	rcStrict = IEMExecOne(pVCpu);
7077	}
7078
7079	if (rcStrict == VINF_IEM_RAISED_XCPT)
7080	{
7081	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7082	rcStrict = VINF_SUCCESS;
7083	}
7084	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7085	return VBOXSTRICTRC_VAL(rcStrict);
7086	}
7087
7088
7089	/**
7090	* \#VMEXIT handler for HLT (SVM_EXIT_HLT). Conditional \#VMEXIT.
7091	*/
7092	HMSVM_EXIT_DECL hmR0SvmExitHlt(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7093	{
7094	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7095
7096	VBOXSTRICTRC rcStrict;
7097	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7098	if (fSupportsNextRipSave)
7099	{
7100	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
7101	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7102	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7103	rcStrict = IEMExecDecodedHlt(pVCpu, cbInstr);
7104	}
7105	else
7106	{
7107	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7108	rcStrict = IEMExecOne(pVCpu);
7109	}
7110
7111	if ( rcStrict == VINF_EM_HALT
7112	\|\| rcStrict == VINF_SUCCESS)
7113	rcStrict = EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx) ? VINF_SUCCESS : VINF_EM_HALT;
7114	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7115	{
7116	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7117	rcStrict = VINF_SUCCESS;
7118	}
7119	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7120	if (rcStrict != VINF_SUCCESS)
7121	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
7122	return VBOXSTRICTRC_VAL(rcStrict);;
7123	}
7124
7125
7126	/**
7127	* \#VMEXIT handler for MONITOR (SVM_EXIT_MONITOR). Conditional \#VMEXIT.
7128	*/
7129	HMSVM_EXIT_DECL hmR0SvmExitMonitor(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7130	{
7131	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7132
7133	/*
7134	* If the instruction length is supplied by the CPU is 3 bytes, we can be certain that no
7135	* segment override prefix is present (and thus use the default segment DS). Otherwise, a
7136	* segment override prefix or other prefixes might be used, in which case we fallback to
7137	* IEMExecOne() to figure out.
7138	*/
7139	VBOXSTRICTRC rcStrict;
7140	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7141	uint8_t const cbInstr = hmR0SvmSupportsNextRipSave(pVCpu) ? pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip : 0;
7142	if (cbInstr)
7143	{
7144	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
7145	rcStrict = IEMExecDecodedMonitor(pVCpu, cbInstr);
7146	}
7147	else
7148	{
7149	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7150	rcStrict = IEMExecOne(pVCpu);
7151	}
7152
7153	if (rcStrict == VINF_IEM_RAISED_XCPT)
7154	{
7155	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7156	rcStrict = VINF_SUCCESS;
7157	}
7158	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7159	return rcStrict;
7160	}
7161
7162
7163	/**
7164	* \#VMEXIT handler for MWAIT (SVM_EXIT_MWAIT). Conditional \#VMEXIT.
7165	*/
7166	HMSVM_EXIT_DECL hmR0SvmExitMwait(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7167	{
7168	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7169
7170	VBOXSTRICTRC rcStrict;
7171	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7172	if (fSupportsNextRipSave)
7173	{
7174	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
7175	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7176	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7177	rcStrict = IEMExecDecodedMwait(pVCpu, cbInstr);
7178	}
7179	else
7180	{
7181	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7182	rcStrict = IEMExecOne(pVCpu);
7183	}
7184
7185	if ( rcStrict == VINF_EM_HALT
7186	&& EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
7187	rcStrict = VINF_SUCCESS;
7188	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7189	{
7190	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7191	rcStrict = VINF_SUCCESS;
7192	}
7193	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7194	return rcStrict;
7195	}
7196
7197
7198	/**
7199	* \#VMEXIT handler for shutdown (triple-fault) (SVM_EXIT_SHUTDOWN). Conditional
7200	* \#VMEXIT.
7201	*/
7202	HMSVM_EXIT_DECL hmR0SvmExitShutdown(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7203	{
7204	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7205	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7206	return VINF_EM_RESET;
7207	}
7208
7209
7210	/**
7211	* \#VMEXIT handler for unexpected exits. Conditional \#VMEXIT.
7212	*/
7213	HMSVM_EXIT_DECL hmR0SvmExitUnexpected(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7214	{
7215	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7216	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7217	AssertMsgFailed(("hmR0SvmExitUnexpected: ExitCode=%#RX64 uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", pSvmTransient->u64ExitCode,
7218	pVmcb->ctrl.u64ExitInfo1, pVmcb->ctrl.u64ExitInfo2));
7219	RT_NOREF(pVmcb);
7220	pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
7221	return VERR_SVM_UNEXPECTED_EXIT;
7222	}
7223
7224
7225	/**
7226	* \#VMEXIT handler for CRx reads (SVM_EXIT_READ_CR*). Conditional \#VMEXIT.
7227	*/
7228	HMSVM_EXIT_DECL hmR0SvmExitReadCRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7229	{
7230	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7231
7232	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7233	Log4Func(("CS:RIP=%04x:%RX64\n", pCtx->cs.Sel, pCtx->rip));
7234	#ifdef VBOX_WITH_STATISTICS
7235	switch (pSvmTransient->u64ExitCode)
7236	{
7237	case SVM_EXIT_READ_CR0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
7238	case SVM_EXIT_READ_CR2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
7239	case SVM_EXIT_READ_CR3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
7240	case SVM_EXIT_READ_CR4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
7241	case SVM_EXIT_READ_CR8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
7242	}
7243	#endif
7244
7245	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
7246	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7247	if ( fSupportsDecodeAssists
7248	&& fSupportsNextRipSave)
7249	{
7250	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7251	bool const fMovCRx = RT_BOOL(pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_MASK);
7252	if (fMovCRx)
7253	{
7254	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR_MASK
7255	\| CPUMCTX_EXTRN_APIC_TPR);
7256	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
7257	uint8_t const iCrReg = pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0;
7258	uint8_t const iGReg = pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_GPR_NUMBER;
7259	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
7260	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7261	return VBOXSTRICTRC_VAL(rcStrict);
7262	}
7263	/* else: SMSW instruction, fall back below to IEM for this. */
7264	}
7265
7266	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7267	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
7268	AssertMsg( rcStrict == VINF_SUCCESS
7269	\|\| rcStrict == VINF_PGM_SYNC_CR3
7270	\|\| rcStrict == VINF_IEM_RAISED_XCPT,
7271	("hmR0SvmExitReadCRx: IEMExecOne failed rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7272	Assert((pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0) <= 15);
7273	if (rcStrict == VINF_IEM_RAISED_XCPT)
7274	{
7275	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7276	rcStrict = VINF_SUCCESS;
7277	}
7278	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7279	return rcStrict;
7280	}
7281
7282
7283	/**
7284	* \#VMEXIT handler for CRx writes (SVM_EXIT_WRITE_CR*). Conditional \#VMEXIT.
7285	*/
7286	HMSVM_EXIT_DECL hmR0SvmExitWriteCRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7287	{
7288	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7289
7290	uint64_t const uExitCode = pSvmTransient->u64ExitCode;
7291	uint8_t const iCrReg = uExitCode == SVM_EXIT_CR0_SEL_WRITE ? 0 : (pSvmTransient->u64ExitCode - SVM_EXIT_WRITE_CR0);
7292	Assert(iCrReg <= 15);
7293
7294	VBOXSTRICTRC rcStrict = VERR_SVM_IPE_5;
7295	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7296	bool fDecodedInstr = false;
7297	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
7298	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7299	if ( fSupportsDecodeAssists
7300	&& fSupportsNextRipSave)
7301	{
7302	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7303	bool const fMovCRx = RT_BOOL(pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_MASK);
7304	if (fMovCRx)
7305	{
7306	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4
7307	\| CPUMCTX_EXTRN_APIC_TPR);
7308	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
7309	uint8_t const iGReg = pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_GPR_NUMBER;
7310	Log4Func(("Mov CR%u w/ iGReg=%#x\n", iCrReg, iGReg));
7311	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
7312	fDecodedInstr = true;
7313	}
7314	/* else: LMSW or CLTS instruction, fall back below to IEM for this. */
7315	}
7316
7317	if (!fDecodedInstr)
7318	{
7319	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7320	Log4Func(("iCrReg=%#x\n", iCrReg));
7321	rcStrict = IEMExecOne(pVCpu);
7322	if (RT_UNLIKELY( rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED
7323	\|\| rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED))
7324	rcStrict = VERR_EM_INTERPRETER;
7325	}
7326
7327	if (rcStrict == VINF_SUCCESS)
7328	{
7329	switch (iCrReg)
7330	{
7331	case 0:
7332	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
7333	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
7334	break;
7335
7336	case 2:
7337	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR2);
7338	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
7339	break;
7340
7341	case 3:
7342	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR3);
7343	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
7344	break;
7345
7346	case 4:
7347	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
7348	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
7349	break;
7350
7351	case 8:
7352	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7353	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
7354	break;
7355
7356	default:
7357	{
7358	AssertMsgFailed(("hmR0SvmExitWriteCRx: Invalid/Unexpected Write-CRx exit. u64ExitCode=%#RX64 %#x\n",
7359	pSvmTransient->u64ExitCode, iCrReg));
7360	break;
7361	}
7362	}
7363	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7364	}
7365	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7366	{
7367	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7368	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7369	rcStrict = VINF_SUCCESS;
7370	}
7371	else
7372	Assert(rcStrict == VERR_EM_INTERPRETER \|\| rcStrict == VINF_PGM_SYNC_CR3);
7373	return rcStrict;
7374	}
7375
7376
7377	/**
7378	* \#VMEXIT helper for read MSRs, see hmR0SvmExitMsr.
7379	*
7380	* @returns Strict VBox status code.
7381	* @param pVCpu The cross context virtual CPU structure.
7382	* @param pVmcb Pointer to the VM control block.
7383	*/
7384	static VBOXSTRICTRC hmR0SvmExitReadMsr(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
7385	{
7386	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7387	Log4Func(("idMsr=%#RX32\n", pVCpu->cpum.GstCtx.ecx));
7388
7389	VBOXSTRICTRC rcStrict;
7390	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7391	if (fSupportsNextRipSave)
7392	{
7393	/** @todo Optimize this: Only retrieve the MSR bits we need here. CPUMAllMsrs.cpp
7394	* can ask for what it needs instead of using CPUMCTX_EXTRN_ALL_MSRS. */
7395	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
7396	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7397	rcStrict = IEMExecDecodedRdmsr(pVCpu, cbInstr);
7398	}
7399	else
7400	{
7401	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
7402	rcStrict = IEMExecOne(pVCpu);
7403	}
7404
7405	AssertMsg( rcStrict == VINF_SUCCESS
7406	\|\| rcStrict == VINF_IEM_RAISED_XCPT
7407	\|\| rcStrict == VINF_CPUM_R3_MSR_READ,
7408	("hmR0SvmExitReadMsr: Unexpected status %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7409
7410	if (rcStrict == VINF_IEM_RAISED_XCPT)
7411	{
7412	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7413	rcStrict = VINF_SUCCESS;
7414	}
7415	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7416	return rcStrict;
7417	}
7418
7419
7420	/**
7421	* \#VMEXIT helper for write MSRs, see hmR0SvmExitMsr.
7422	*
7423	* @returns Strict VBox status code.
7424	* @param pVCpu The cross context virtual CPU structure.
7425	* @param pVmcb Pointer to the VM control block.
7426	* @param pSvmTransient Pointer to the SVM-transient structure.
7427	*/
7428	static VBOXSTRICTRC hmR0SvmExitWriteMsr(PVMCPUCC pVCpu, PSVMVMCB pVmcb, PSVMTRANSIENT pSvmTransient)
7429	{
7430	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7431	uint32_t const idMsr = pCtx->ecx;
7432	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7433	Log4Func(("idMsr=%#RX32\n", idMsr));
7434
7435	/*
7436	* Handle TPR patching MSR writes.
7437	* We utilitize the LSTAR MSR for patching.
7438	*/
7439	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7440	if ( idMsr == MSR_K8_LSTAR
7441	&& pVCpu->CTX_SUFF(pVM)->hm.s.fTprPatchingActive)
7442	{
7443	unsigned cbInstr;
7444	if (fSupportsNextRipSave)
7445	cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7446	else
7447	{
7448	PDISCPUSTATE pDis = &pVCpu->hmr0.s.svm.DisState;
7449	int rc = EMInterpretDisasCurrent(pVCpu->CTX_SUFF(pVM), pVCpu, pDis, &cbInstr);
7450	if ( rc == VINF_SUCCESS
7451	&& pDis->pCurInstr->uOpcode == OP_WRMSR)
7452	Assert(cbInstr > 0);
7453	else
7454	cbInstr = 0;
7455	}
7456
7457	/* Our patch code uses LSTAR for TPR caching for 32-bit guests. */
7458	if ((pCtx->eax & 0xff) != pSvmTransient->u8GuestTpr)
7459	{
7460	int rc = APICSetTpr(pVCpu, pCtx->eax & 0xff);
7461	AssertRCReturn(rc, rc);
7462	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7463	}
7464
7465	int rc = VINF_SUCCESS;
7466	hmR0SvmAdvanceRip(pVCpu, cbInstr);
7467	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
7468	return rc;
7469	}
7470
7471	/*
7472	* Handle regular MSR writes.
7473	*/
7474	VBOXSTRICTRC rcStrict;
7475	if (fSupportsNextRipSave)
7476	{
7477	/** @todo Optimize this: We don't need to get much of the MSR state here
7478	* since we're only updating. CPUMAllMsrs.cpp can ask for what it needs and
7479	* clear the applicable extern flags. */
7480	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
7481	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7482	rcStrict = IEMExecDecodedWrmsr(pVCpu, cbInstr);
7483	}
7484	else
7485	{
7486	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
7487	rcStrict = IEMExecOne(pVCpu);
7488	}
7489
7490	AssertMsg( rcStrict == VINF_SUCCESS
7491	\|\| rcStrict == VINF_IEM_RAISED_XCPT
7492	\|\| rcStrict == VINF_CPUM_R3_MSR_WRITE,
7493	("hmR0SvmExitWriteMsr: Unexpected status %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7494
7495	if (rcStrict == VINF_SUCCESS)
7496	{
7497	/* If this is an X2APIC WRMSR access, update the APIC TPR state. */
7498	if ( idMsr >= MSR_IA32_X2APIC_START
7499	&& idMsr <= MSR_IA32_X2APIC_END)
7500	{
7501	/*
7502	* We've already saved the APIC related guest-state (TPR) in hmR0SvmPostRunGuest().
7503	* When full APIC register virtualization is implemented we'll have to make sure
7504	* APIC state is saved from the VMCB before IEM changes it.
7505	*/
7506	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7507	}
7508	else
7509	{
7510	switch (idMsr)
7511	{
7512	case MSR_IA32_TSC: pSvmTransient->fUpdateTscOffsetting = true; break;
7513	case MSR_K6_EFER: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR); break;
7514	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
7515	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
7516	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
7517	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
7518	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
7519	}
7520	}
7521	}
7522	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7523	{
7524	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7525	rcStrict = VINF_SUCCESS;
7526	}
7527	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7528	return rcStrict;
7529	}
7530
7531
7532	/**
7533	* \#VMEXIT handler for MSR read and writes (SVM_EXIT_MSR). Conditional
7534	* \#VMEXIT.
7535	*/
7536	HMSVM_EXIT_DECL hmR0SvmExitMsr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7537	{
7538	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7539
7540	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7541	if (pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_READ)
7542	return hmR0SvmExitReadMsr(pVCpu, pVmcb);
7543
7544	Assert(pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_WRITE);
7545	return hmR0SvmExitWriteMsr(pVCpu, pVmcb, pSvmTransient);
7546	}
7547
7548
7549	/**
7550	* \#VMEXIT handler for DRx read (SVM_EXIT_READ_DRx). Conditional \#VMEXIT.
7551	*/
7552	HMSVM_EXIT_DECL hmR0SvmExitReadDRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7553	{
7554	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7555	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7556
7557	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
7558
7559	/** @todo Stepping with nested-guest. */
7560	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7561	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
7562	{
7563	/* We should -not- get this #VMEXIT if the guest's debug registers were active. */
7564	if (pSvmTransient->fWasGuestDebugStateActive)
7565	{
7566	AssertMsgFailed(("hmR0SvmExitReadDRx: Unexpected exit %#RX32\n", (uint32_t)pSvmTransient->u64ExitCode));
7567	pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
7568	return VERR_SVM_UNEXPECTED_EXIT;
7569	}
7570
7571	/*
7572	* Lazy DR0-3 loading.
7573	*/
7574	if (!pSvmTransient->fWasHyperDebugStateActive)
7575	{
7576	Assert(!DBGFIsStepping(pVCpu)); Assert(!pVCpu->hm.s.fSingleInstruction);
7577	Log5(("hmR0SvmExitReadDRx: Lazy loading guest debug registers\n"));
7578
7579	/* Don't intercept DRx read and writes. */
7580	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
7581	pVmcb->ctrl.u16InterceptRdDRx = 0;
7582	pVmcb->ctrl.u16InterceptWrDRx = 0;
7583	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
7584
7585	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
7586	VMMRZCallRing3Disable(pVCpu);
7587	HM_DISABLE_PREEMPT(pVCpu);
7588
7589	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
7590	CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
7591	Assert(CPUMIsGuestDebugStateActive(pVCpu));
7592
7593	HM_RESTORE_PREEMPT();
7594	VMMRZCallRing3Enable(pVCpu);
7595
7596	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
7597	return VINF_SUCCESS;
7598	}
7599	}
7600
7601	/*
7602	* Interpret the read/writing of DRx.
7603	*/
7604	/** @todo Decode assist. */
7605	VBOXSTRICTRC rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0 /* pvFault */);
7606	Log5(("hmR0SvmExitReadDRx: Emulated DRx access: rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
7607	if (RT_LIKELY(rc == VINF_SUCCESS))
7608	{
7609	/* Not necessary for read accesses but whatever doesn't hurt for now, will be fixed with decode assist. */
7610	/** @todo CPUM should set this flag! */
7611	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR_MASK);
7612	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
7613	}
7614	else
7615	Assert(rc == VERR_EM_INTERPRETER);
7616	return rc;
7617	}
7618
7619
7620	/**
7621	* \#VMEXIT handler for DRx write (SVM_EXIT_WRITE_DRx). Conditional \#VMEXIT.
7622	*/
7623	HMSVM_EXIT_DECL hmR0SvmExitWriteDRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7624	{
7625	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7626	/* For now it's the same since we interpret the instruction anyway. Will change when using of Decode Assist is implemented. */
7627	VBOXSTRICTRC rc = hmR0SvmExitReadDRx(pVCpu, pSvmTransient);
7628	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
7629	STAM_COUNTER_DEC(&pVCpu->hm.s.StatExitDRxRead);
7630	return rc;
7631	}
7632
7633
7634	/**
7635	* \#VMEXIT handler for XCRx write (SVM_EXIT_XSETBV). Conditional \#VMEXIT.
7636	*/
7637	HMSVM_EXIT_DECL hmR0SvmExitXsetbv(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7638	{
7639	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7640	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7641
7642	/** @todo decode assists... */
7643	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
7644	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
7645	{
7646	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7647	bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
7648	Log4Func(("New XCR0=%#RX64 fLoadSaveGuestXcr0=%RTbool (cr4=%#RX64)\n", pCtx->aXcr[0], fLoadSaveGuestXcr0, pCtx->cr4));
7649	if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
7650	{
7651	pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
7652	hmR0SvmUpdateVmRunFunction(pVCpu);
7653	}
7654	}
7655	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7656	{
7657	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7658	rcStrict = VINF_SUCCESS;
7659	}
7660	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7661	return rcStrict;
7662	}
7663
7664
7665	/**
7666	* \#VMEXIT handler for I/O instructions (SVM_EXIT_IOIO). Conditional \#VMEXIT.
7667	*/
7668	HMSVM_EXIT_DECL hmR0SvmExitIOInstr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7669	{
7670	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7671	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK);
7672
7673	/* I/O operation lookup arrays. */
7674	static uint32_t const s_aIOSize[8] = { 0, 1, 2, 0, 4, 0, 0, 0 }; /* Size of the I/O accesses in bytes. */
7675	static uint32_t const s_aIOOpAnd[8] = { 0, 0xff, 0xffff, 0, 0xffffffff, 0, 0, 0 }; /* AND masks for saving
7676	the result (in AL/AX/EAX). */
7677	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7678	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7679	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7680
7681	Log4Func(("CS:RIP=%04x:%RX64\n", pCtx->cs.Sel, pCtx->rip));
7682
7683	/* Refer AMD spec. 15.10.2 "IN and OUT Behaviour" and Figure 15-2. "EXITINFO1 for IOIO Intercept" for the format. */
7684	SVMIOIOEXITINFO IoExitInfo;
7685	IoExitInfo.u = (uint32_t)pVmcb->ctrl.u64ExitInfo1;
7686	uint32_t uIOWidth = (IoExitInfo.u >> 4) & 0x7;
7687	uint32_t cbValue = s_aIOSize[uIOWidth];
7688	uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
7689
7690	if (RT_UNLIKELY(!cbValue))
7691	{
7692	AssertMsgFailed(("hmR0SvmExitIOInstr: Invalid IO operation. uIOWidth=%u\n", uIOWidth));
7693	return VERR_EM_INTERPRETER;
7694	}
7695
7696	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
7697	VBOXSTRICTRC rcStrict;
7698	PCEMEXITREC pExitRec = NULL;
7699	if ( !pVCpu->hm.s.fSingleInstruction
7700	&& !pVCpu->cpum.GstCtx.eflags.Bits.u1TF)
7701	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
7702	!IoExitInfo.n.u1Str
7703	? IoExitInfo.n.u1Type == SVM_IOIO_READ
7704	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
7705	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
7706	: IoExitInfo.n.u1Type == SVM_IOIO_READ
7707	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
7708	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
7709	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
7710	if (!pExitRec)
7711	{
7712	bool fUpdateRipAlready = false;
7713	if (IoExitInfo.n.u1Str)
7714	{
7715	/* INS/OUTS - I/O String instruction. */
7716	/** @todo Huh? why can't we use the segment prefix information given by AMD-V
7717	* in EXITINFO1? Investigate once this thing is up and running. */
7718	Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, IoExitInfo.n.u16Port, cbValue,
7719	IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? 'w' : 'r'));
7720	AssertReturn(pCtx->dx == IoExitInfo.n.u16Port, VERR_SVM_IPE_2);
7721	static IEMMODE const s_aenmAddrMode[8] =
7722	{
7723	(IEMMODE)-1, IEMMODE_16BIT, IEMMODE_32BIT, (IEMMODE)-1, IEMMODE_64BIT, (IEMMODE)-1, (IEMMODE)-1, (IEMMODE)-1
7724	};
7725	IEMMODE enmAddrMode = s_aenmAddrMode[(IoExitInfo.u >> 7) & 0x7];
7726	if (enmAddrMode != (IEMMODE)-1)
7727	{
7728	uint64_t cbInstr = pVmcb->ctrl.u64ExitInfo2 - pCtx->rip;
7729	if (cbInstr <= 15 && cbInstr >= 1)
7730	{
7731	Assert(cbInstr >= 1U + IoExitInfo.n.u1Rep);
7732	if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
7733	{
7734	/* Don't know exactly how to detect whether u3Seg is valid, currently
7735	only enabling it for Bulldozer and later with NRIP. OS/2 broke on
7736	2384 Opterons when only checking NRIP. */
7737	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7738	if ( fSupportsNextRipSave
7739	&& pVM->cpum.ro.GuestFeatures.enmMicroarch >= kCpumMicroarch_AMD_15h_First)
7740	{
7741	AssertMsg(IoExitInfo.n.u3Seg == X86_SREG_DS \|\| cbInstr > 1U + IoExitInfo.n.u1Rep,
7742	("u32Seg=%d cbInstr=%d u1REP=%d", IoExitInfo.n.u3Seg, cbInstr, IoExitInfo.n.u1Rep));
7743	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
7744	IoExitInfo.n.u3Seg, true /fIoChecked/);
7745	}
7746	else if (cbInstr == 1U + IoExitInfo.n.u1Rep)
7747	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
7748	X86_SREG_DS, true /fIoChecked/);
7749	else
7750	rcStrict = IEMExecOne(pVCpu);
7751	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7752	}
7753	else
7754	{
7755	AssertMsg(IoExitInfo.n.u3Seg == X86_SREG_ES /=0/, ("%#x\n", IoExitInfo.n.u3Seg));
7756	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
7757	true /fIoChecked/);
7758	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7759	}
7760	}
7761	else
7762	{
7763	AssertMsgFailed(("rip=%RX64 nrip=%#RX64 cbInstr=%#RX64\n", pCtx->rip, pVmcb->ctrl.u64ExitInfo2, cbInstr));
7764	rcStrict = IEMExecOne(pVCpu);
7765	}
7766	}
7767	else
7768	{
7769	AssertMsgFailed(("IoExitInfo=%RX64\n", IoExitInfo.u));
7770	rcStrict = IEMExecOne(pVCpu);
7771	}
7772	fUpdateRipAlready = true;
7773	if (rcStrict == VINF_IEM_RAISED_XCPT)
7774	{
7775	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7776	rcStrict = VINF_SUCCESS;
7777	}
7778	}
7779	else
7780	{
7781	/* IN/OUT - I/O instruction. */
7782	Assert(!IoExitInfo.n.u1Rep);
7783
7784	uint8_t const cbInstr = pVmcb->ctrl.u64ExitInfo2 - pCtx->rip;
7785	if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
7786	{
7787	rcStrict = IOMIOPortWrite(pVM, pVCpu, IoExitInfo.n.u16Port, pCtx->eax & uAndVal, cbValue);
7788	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
7789	&& !pCtx->eflags.Bits.u1TF)
7790	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, IoExitInfo.n.u16Port, cbInstr, cbValue, pCtx->eax & uAndVal);
7791	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
7792	}
7793	else
7794	{
7795	uint32_t u32Val = 0;
7796	rcStrict = IOMIOPortRead(pVM, pVCpu, IoExitInfo.n.u16Port, &u32Val, cbValue);
7797	if (IOM_SUCCESS(rcStrict))
7798	{
7799	/* Save result of I/O IN instr. in AL/AX/EAX. */
7800	/** @todo r=bird: 32-bit op size should clear high bits of rax! */
7801	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
7802	}
7803	else if ( rcStrict == VINF_IOM_R3_IOPORT_READ
7804	&& !pCtx->eflags.Bits.u1TF)
7805	rcStrict = EMRZSetPendingIoPortRead(pVCpu, IoExitInfo.n.u16Port, cbInstr, cbValue);
7806
7807	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
7808	}
7809	}
7810
7811	if (IOM_SUCCESS(rcStrict))
7812	{
7813	/* AMD-V saves the RIP of the instruction following the IO instruction in EXITINFO2. */
7814	if (!fUpdateRipAlready)
7815	pCtx->rip = pVmcb->ctrl.u64ExitInfo2;
7816
7817	/*
7818	* If any I/O breakpoints are armed, we need to check if one triggered
7819	* and take appropriate action.
7820	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
7821	*/
7822	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
7823	* execution engines about whether hyper BPs and such are pending. */
7824	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_DR7);
7825	uint32_t const uDr7 = pCtx->dr[7];
7826	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
7827	&& X86_DR7_ANY_RW_IO(uDr7)
7828	&& (pCtx->cr4 & X86_CR4_DE))
7829	\|\| DBGFBpIsHwIoArmed(pVM)))
7830	{
7831	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
7832	VMMRZCallRing3Disable(pVCpu);
7833	HM_DISABLE_PREEMPT(pVCpu);
7834
7835	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
7836	CPUMR0DebugStateMaybeSaveGuest(pVCpu, false /fDr6/);
7837
7838	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, &pVCpu->cpum.GstCtx, IoExitInfo.n.u16Port, cbValue);
7839	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
7840	{
7841	/* Raise #DB. */
7842	pVmcb->guest.u64DR6 = pCtx->dr[6];
7843	pVmcb->guest.u64DR7 = pCtx->dr[7];
7844	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
7845	hmR0SvmSetPendingXcptDB(pVCpu);
7846	}
7847	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
7848	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
7849	else if ( rcStrict2 != VINF_SUCCESS
7850	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
7851	rcStrict = rcStrict2;
7852	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
7853
7854	HM_RESTORE_PREEMPT();
7855	VMMRZCallRing3Enable(pVCpu);
7856	}
7857
7858	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7859	}
7860	#ifdef VBOX_STRICT
7861	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
7862	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
7863	Assert(IoExitInfo.n.u1Type == SVM_IOIO_READ);
7864	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
7865	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
7866	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
7867	Assert(IoExitInfo.n.u1Type == SVM_IOIO_WRITE);
7868	else
7869	{
7870	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
7871	* statuses, that the VMM device and some others may return. See
7872	* IOM_SUCCESS() for guidance. */
7873	AssertMsg( RT_FAILURE(rcStrict)
7874	\|\| rcStrict == VINF_SUCCESS
7875	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
7876	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
7877	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
7878	\|\| rcStrict == VINF_EM_DBG_STEPPED
7879	\|\| rcStrict == VINF_EM_RAW_TO_R3
7880	\|\| rcStrict == VINF_EM_TRIPLE_FAULT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7881	}
7882	#endif
7883	}
7884	else
7885	{
7886	/*
7887	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
7888	*/
7889	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7890	STAM_COUNTER_INC(!IoExitInfo.n.u1Str
7891	? IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
7892	: IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
7893	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
7894	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, IoExitInfo.n.u1Rep ? "REP " : "",
7895	IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? "OUT" : "IN", IoExitInfo.n.u1Str ? "S" : "", IoExitInfo.n.u16Port, uIOWidth));
7896
7897	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
7898	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7899
7900	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
7901	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
7902	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
7903	}
7904	return rcStrict;
7905	}
7906
7907
7908	/**
7909	* \#VMEXIT handler for Nested Page-faults (SVM_EXIT_NPF). Conditional \#VMEXIT.
7910	*/
7911	HMSVM_EXIT_DECL hmR0SvmExitNestedPF(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7912	{
7913	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7914	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7915	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
7916
7917	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7918	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7919	Assert(pVM->hmr0.s.fNestedPaging);
7920
7921	/* See AMD spec. 15.25.6 "Nested versus Guest Page Faults, Fault Ordering" for VMCB details for #NPF. */
7922	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7923	RTGCPHYS GCPhysFaultAddr = pVmcb->ctrl.u64ExitInfo2;
7924	uint32_t u32ErrCode = pVmcb->ctrl.u64ExitInfo1; /* Note! High bits in EXITINFO1 may contain additional info and are
7925	thus intentionally not copied into u32ErrCode. */
7926
7927	Log4Func(("#NPF at CS:RIP=%04x:%RX64 GCPhysFaultAddr=%RGp ErrCode=%#x cbInstrFetched=%u %.15Rhxs\n", pCtx->cs.Sel, pCtx->rip, GCPhysFaultAddr,
7928	u32ErrCode, pVmcb->ctrl.cbInstrFetched, pVmcb->ctrl.abInstr));
7929
7930	/*
7931	* TPR patching for 32-bit guests, using the reserved bit in the page tables for MMIO regions.
7932	*/
7933	if ( pVM->hm.s.fTprPatchingAllowed
7934	&& (GCPhysFaultAddr & GUEST_PAGE_OFFSET_MASK) == XAPIC_OFF_TPR
7935	&& ( !(u32ErrCode & X86_TRAP_PF_P) /* Not present */
7936	\|\| (u32ErrCode & (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) == (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) /* MMIO page. */
7937	&& !CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
7938	&& !CPUMIsGuestInLongModeEx(pCtx)
7939	&& !CPUMGetGuestCPL(pVCpu)
7940	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
7941	{
7942	RTGCPHYS GCPhysApicBase = APICGetBaseMsrNoCheck(pVCpu);
7943	GCPhysApicBase &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
7944
7945	if (GCPhysFaultAddr == GCPhysApicBase + XAPIC_OFF_TPR)
7946	{
7947	/* Only attempt to patch the instruction once. */
7948	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
7949	if (!pPatch)
7950	return VINF_EM_HM_PATCH_TPR_INSTR;
7951	}
7952	}
7953
7954	/*
7955	* Determine the nested paging mode.
7956	*/
7957	/** @todo r=bird: Gotta love this nested paging hacking we're still carrying with us... (Split PGM_TYPE_NESTED.) */
7958	PGMMODE const enmNestedPagingMode = PGMGetHostMode(pVM);
7959
7960	/*
7961	* MMIO optimization using the reserved (RSVD) bit in the guest page tables for MMIO pages.
7962	*/
7963	Assert((u32ErrCode & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) != X86_TRAP_PF_RSVD);
7964	if ((u32ErrCode & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
7965	{
7966	/*
7967	* If event delivery causes an MMIO #NPF, go back to instruction emulation as otherwise
7968	* injecting the original pending event would most likely cause the same MMIO #NPF.
7969	*/
7970	if (pVCpu->hm.s.Event.fPending)
7971	{
7972	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
7973	return VINF_EM_RAW_INJECT_TRPM_EVENT;
7974	}
7975
7976	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
7977	VBOXSTRICTRC rcStrict;
7978	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
7979	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
7980	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
7981	if (!pExitRec)
7982	{
7983
7984	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, enmNestedPagingMode, CPUMCTX2CORE(pCtx), GCPhysFaultAddr,
7985	u32ErrCode);
7986
7987	/*
7988	* If we succeed, resume guest execution.
7989	*
7990	* If we fail in interpreting the instruction because we couldn't get the guest
7991	* physical address of the page containing the instruction via the guest's page
7992	* tables (we would invalidate the guest page in the host TLB), resume execution
7993	* which would cause a guest page fault to let the guest handle this weird case.
7994	*
7995	* See @bugref{6043}.
7996	*/
7997	if ( rcStrict == VINF_SUCCESS
7998	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
7999	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
8000	{
8001	/* Successfully handled MMIO operation. */
8002	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8003	rcStrict = VINF_SUCCESS;
8004	}
8005	}
8006	else
8007	{
8008	/*
8009	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
8010	*/
8011	Assert(pCtx == &pVCpu->cpum.GstCtx);
8012	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8013	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
8014	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhysFaultAddr));
8015
8016	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
8017	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8018
8019	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
8020	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
8021	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
8022	}
8023	return rcStrict;
8024	}
8025
8026	/*
8027	* Nested page-fault.
8028	*/
8029	TRPMAssertXcptPF(pVCpu, GCPhysFaultAddr, u32ErrCode);
8030	int rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, enmNestedPagingMode, u32ErrCode, CPUMCTX2CORE(pCtx), GCPhysFaultAddr);
8031	TRPMResetTrap(pVCpu);
8032
8033	Log4Func(("#NPF: PGMR0Trap0eHandlerNestedPaging returns %Rrc CS:RIP=%04x:%RX64\n", rc, pCtx->cs.Sel, pCtx->rip));
8034
8035	/*
8036	* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}.
8037	*/
8038	if ( rc == VINF_SUCCESS
8039	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8040	\|\| rc == VERR_PAGE_NOT_PRESENT)
8041	{
8042	/* We've successfully synced our shadow page tables. */
8043	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8044	rc = VINF_SUCCESS;
8045	}
8046
8047	/*
8048	* If delivering an event causes an #NPF (and not MMIO), we shall resolve the fault and
8049	* re-inject the original event.
8050	*/
8051	if (pVCpu->hm.s.Event.fPending)
8052	{
8053	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflectNPF);
8054
8055	/*
8056	* If the #NPF handler requested emulation of the instruction, ignore it.
8057	* We need to re-inject the original event so as to not lose it.
8058	* Reproducible when booting ReactOS 0.4.12 with BTRFS (installed using BootCD,
8059	* LiveCD is broken for other reasons).
8060	*/
8061	if (rc == VINF_EM_RAW_EMULATE_INSTR)
8062	rc = VINF_EM_RAW_INJECT_TRPM_EVENT;
8063	}
8064
8065	return rc;
8066	}
8067
8068
8069	/**
8070	* \#VMEXIT handler for virtual interrupt (SVM_EXIT_VINTR). Conditional
8071	* \#VMEXIT.
8072	*/
8073	HMSVM_EXIT_DECL hmR0SvmExitVIntr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8074	{
8075	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8076	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
8077
8078	/* Indicate that we no longer need to #VMEXIT when the guest is ready to receive NMIs, it is now ready. */
8079	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8080	hmR0SvmClearIntWindowExiting(pVCpu, pVmcb);
8081
8082	/* Deliver the pending interrupt via hmR0SvmEvaluatePendingEvent() and resume guest execution. */
8083	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
8084	return VINF_SUCCESS;
8085	}
8086
8087
8088	/**
8089	* \#VMEXIT handler for task switches (SVM_EXIT_TASK_SWITCH). Conditional
8090	* \#VMEXIT.
8091	*/
8092	HMSVM_EXIT_DECL hmR0SvmExitTaskSwitch(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8093	{
8094	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8095	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8096
8097	#ifndef HMSVM_ALWAYS_TRAP_TASK_SWITCH
8098	Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
8099	#endif
8100
8101	/* Check if this task-switch occurred while delivering an event through the guest IDT. */
8102	if (pVCpu->hm.s.Event.fPending) /* Can happen with exceptions/NMI. See @bugref{8411}. */
8103	{
8104	/*
8105	* AMD-V provides us with the exception which caused the TS; we collect
8106	* the information in the call to hmR0SvmCheckExitDueToEventDelivery().
8107	*/
8108	Log4Func(("TS occurred during event delivery\n"));
8109	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8110	return VINF_EM_RAW_INJECT_TRPM_EVENT;
8111	}
8112
8113	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8114	* emulation. */
8115	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8116	return VERR_EM_INTERPRETER;
8117	}
8118
8119
8120	/**
8121	* \#VMEXIT handler for VMMCALL (SVM_EXIT_VMMCALL). Conditional \#VMEXIT.
8122	*/
8123	HMSVM_EXIT_DECL hmR0SvmExitVmmCall(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8124	{
8125	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8126	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8127
8128	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8129	if (pVM->hm.s.fTprPatchingAllowed)
8130	{
8131	int rc = hmEmulateSvmMovTpr(pVM, pVCpu);
8132	if (rc != VERR_NOT_FOUND)
8133	{
8134	Log4Func(("hmEmulateSvmMovTpr returns %Rrc\n", rc));
8135	return rc;
8136	}
8137	}
8138
8139	if (EMAreHypercallInstructionsEnabled(pVCpu))
8140	{
8141	unsigned cbInstr;
8142	if (hmR0SvmSupportsNextRipSave(pVCpu))
8143	{
8144	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8145	cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8146	}
8147	else
8148	{
8149	PDISCPUSTATE pDis = &pVCpu->hmr0.s.svm.DisState;
8150	int rc = EMInterpretDisasCurrent(pVCpu->CTX_SUFF(pVM), pVCpu, pDis, &cbInstr);
8151	if ( rc == VINF_SUCCESS
8152	&& pDis->pCurInstr->uOpcode == OP_VMMCALL)
8153	Assert(cbInstr > 0);
8154	else
8155	cbInstr = 0;
8156	}
8157
8158	VBOXSTRICTRC rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
8159	if (RT_SUCCESS(rcStrict))
8160	{
8161	/* Only update the RIP if we're continuing guest execution and not in the case
8162	of say VINF_GIM_R3_HYPERCALL. */
8163	if (rcStrict == VINF_SUCCESS)
8164	hmR0SvmAdvanceRip(pVCpu, cbInstr);
8165
8166	return VBOXSTRICTRC_VAL(rcStrict);
8167	}
8168	else
8169	Log4Func(("GIMHypercall returns %Rrc -> #UD\n", VBOXSTRICTRC_VAL(rcStrict)));
8170	}
8171
8172	hmR0SvmSetPendingXcptUD(pVCpu);
8173	return VINF_SUCCESS;
8174	}
8175
8176
8177	/**
8178	* \#VMEXIT handler for VMMCALL (SVM_EXIT_VMMCALL). Conditional \#VMEXIT.
8179	*/
8180	HMSVM_EXIT_DECL hmR0SvmExitPause(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8181	{
8182	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8183
8184	unsigned cbInstr;
8185	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8186	if (fSupportsNextRipSave)
8187	{
8188	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8189	cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8190	}
8191	else
8192	{
8193	PDISCPUSTATE pDis = &pVCpu->hmr0.s.svm.DisState;
8194	int rc = EMInterpretDisasCurrent(pVCpu->CTX_SUFF(pVM), pVCpu, pDis, &cbInstr);
8195	if ( rc == VINF_SUCCESS
8196	&& pDis->pCurInstr->uOpcode == OP_PAUSE)
8197	Assert(cbInstr > 0);
8198	else
8199	cbInstr = 0;
8200	}
8201
8202	/** @todo The guest has likely hit a contended spinlock. We might want to
8203	* poke a schedule different guest VCPU. */
8204	hmR0SvmAdvanceRip(pVCpu, cbInstr);
8205	return VINF_EM_RAW_INTERRUPT;
8206	}
8207
8208
8209	/**
8210	* \#VMEXIT handler for FERR intercept (SVM_EXIT_FERR_FREEZE). Conditional
8211	* \#VMEXIT.
8212	*/
8213	HMSVM_EXIT_DECL hmR0SvmExitFerrFreeze(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8214	{
8215	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8216	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0);
8217	Assert(!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE));
8218
8219	Log4Func(("Raising IRQ 13 in response to #FERR\n"));
8220	return PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13 /* u8Irq /, 1 / u8Level /, 0 / uTagSrc */);
8221	}
8222
8223
8224	/**
8225	* \#VMEXIT handler for IRET (SVM_EXIT_IRET). Conditional \#VMEXIT.
8226	*/
8227	HMSVM_EXIT_DECL hmR0SvmExitIret(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8228	{
8229	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8230
8231	/* Indicate that we no longer need to #VMEXIT when the guest is ready to receive NMIs, it is now (almost) ready. */
8232	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8233	hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_IRET);
8234
8235	/* Emulate the IRET. We have to execute the IRET before an NMI, but must potentially
8236	* deliver a pending NMI right after. If the IRET faults, an NMI can come before the
8237	* handler executes. Yes, x86 is ugly.
8238	*/
8239	return VINF_EM_RAW_EMULATE_INSTR;
8240	}
8241
8242
8243	/**
8244	* \#VMEXIT handler for page-fault exceptions (SVM_EXIT_XCPT_14).
8245	* Conditional \#VMEXIT.
8246	*/
8247	HMSVM_EXIT_DECL hmR0SvmExitXcptPF(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8248	{
8249	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8250	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8251	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8252
8253	/* See AMD spec. 15.12.15 "#PF (Page Fault)". */
8254	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8255	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8256	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8257	uint32_t uErrCode = pVmcb->ctrl.u64ExitInfo1;
8258	uint64_t const uFaultAddress = pVmcb->ctrl.u64ExitInfo2;
8259
8260	#if defined(HMSVM_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMSVM_ALWAYS_TRAP_PF)
8261	if (pVM->hmr0.s.fNestedPaging)
8262	{
8263	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
8264	if ( !pSvmTransient->fVectoringDoublePF
8265	\|\| CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
8266	{
8267	/* A genuine guest #PF, reflect it to the guest. */
8268	hmR0SvmSetPendingXcptPF(pVCpu, uErrCode, uFaultAddress);
8269	Log4Func(("#PF: Guest page fault at %04X:%RGv FaultAddr=%RX64 ErrCode=%#x\n", pCtx->cs.Sel, (RTGCPTR)pCtx->rip,
8270	uFaultAddress, uErrCode));
8271	}
8272	else
8273	{
8274	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8275	hmR0SvmSetPendingXcptDF(pVCpu);
8276	Log4Func(("Pending #DF due to vectoring #PF. NP\n"));
8277	}
8278	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8279	return VINF_SUCCESS;
8280	}
8281	#endif
8282
8283	Assert(!pVM->hmr0.s.fNestedPaging);
8284
8285	/*
8286	* TPR patching shortcut for APIC TPR reads and writes; only applicable to 32-bit guests.
8287	*/
8288	if ( pVM->hm.s.fTprPatchingAllowed
8289	&& (uFaultAddress & 0xfff) == XAPIC_OFF_TPR
8290	&& !(uErrCode & X86_TRAP_PF_P) /* Not present. */
8291	&& !CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
8292	&& !CPUMIsGuestInLongModeEx(pCtx)
8293	&& !CPUMGetGuestCPL(pVCpu)
8294	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8295	{
8296	RTGCPHYS GCPhysApicBase;
8297	GCPhysApicBase = APICGetBaseMsrNoCheck(pVCpu);
8298	GCPhysApicBase &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
8299
8300	/* Check if the page at the fault-address is the APIC base. */
8301	PGMPTWALK Walk;
8302	int rc2 = PGMGstGetPage(pVCpu, (RTGCPTR)uFaultAddress, &Walk);
8303	if ( rc2 == VINF_SUCCESS
8304	&& Walk.GCPhys == GCPhysApicBase)
8305	{
8306	/* Only attempt to patch the instruction once. */
8307	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
8308	if (!pPatch)
8309	return VINF_EM_HM_PATCH_TPR_INSTR;
8310	}
8311	}
8312
8313	Log4Func(("#PF: uFaultAddress=%#RX64 CS:RIP=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", uFaultAddress, pCtx->cs.Sel,
8314	pCtx->rip, uErrCode, pCtx->cr3));
8315
8316	/*
8317	* If it's a vectoring #PF, emulate injecting the original event injection as
8318	* PGMTrap0eHandler() is incapable of differentiating between instruction emulation and
8319	* event injection that caused a #PF. See @bugref{6607}.
8320	*/
8321	if (pSvmTransient->fVectoringPF)
8322	{
8323	Assert(pVCpu->hm.s.Event.fPending);
8324	return VINF_EM_RAW_INJECT_TRPM_EVENT;
8325	}
8326
8327	TRPMAssertXcptPF(pVCpu, uFaultAddress, uErrCode);
8328	int rc = PGMTrap0eHandler(pVCpu, uErrCode, CPUMCTX2CORE(pCtx), (RTGCPTR)uFaultAddress);
8329
8330	Log4Func(("#PF: rc=%Rrc\n", rc));
8331
8332	if (rc == VINF_SUCCESS)
8333	{
8334	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
8335	TRPMResetTrap(pVCpu);
8336	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8337	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8338	return rc;
8339	}
8340
8341	if (rc == VINF_EM_RAW_GUEST_TRAP)
8342	{
8343	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
8344
8345	/*
8346	* If a nested-guest delivers a #PF and that causes a #PF which is -not- a shadow #PF,
8347	* we should simply forward the #PF to the guest and is up to the nested-hypervisor to
8348	* determine whether it is a nested-shadow #PF or a #DF, see @bugref{7243#c121}.
8349	*/
8350	if ( !pSvmTransient->fVectoringDoublePF
8351	\|\| CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
8352	{
8353	/* It's a guest (or nested-guest) page fault and needs to be reflected. */
8354	uErrCode = TRPMGetErrorCode(pVCpu); /* The error code might have been changed. */
8355	TRPMResetTrap(pVCpu);
8356
8357	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
8358	/* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
8359	if ( CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
8360	&& CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_PF))
8361	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_XCPT_PF, uErrCode, uFaultAddress);
8362	#endif
8363
8364	hmR0SvmSetPendingXcptPF(pVCpu, uErrCode, uFaultAddress);
8365	}
8366	else
8367	{
8368	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8369	TRPMResetTrap(pVCpu);
8370	hmR0SvmSetPendingXcptDF(pVCpu);
8371	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
8372	}
8373
8374	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8375	return VINF_SUCCESS;
8376	}
8377
8378	TRPMResetTrap(pVCpu);
8379	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
8380	return rc;
8381	}
8382
8383
8384
8385	/**
8386	* \#VMEXIT handler for division overflow exceptions (SVM_EXIT_XCPT_1).
8387	* Conditional \#VMEXIT.
8388	*/
8389	HMSVM_EXIT_DECL hmR0SvmExitXcptDE(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8390	{
8391	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8392	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
8393	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
8394
8395	/* Paranoia; Ensure we cannot be called as a result of event delivery. */
8396	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8397	Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
8398
8399	int rc = VERR_SVM_UNEXPECTED_XCPT_EXIT;
8400	if (pVCpu->hm.s.fGCMTrapXcptDE)
8401	{
8402	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8403	uint8_t cbInstr = 0;
8404	VBOXSTRICTRC rcStrict = GCMXcptDE(pVCpu, &pVCpu->cpum.GstCtx, NULL /* pDis */, &cbInstr);
8405	if (rcStrict == VINF_SUCCESS)
8406	rc = VINF_SUCCESS; /* Restart instruction with modified guest register context. */
8407	else if (rcStrict == VERR_NOT_FOUND)
8408	rc = VERR_NOT_FOUND; /* Deliver the exception. */
8409	else
8410	Assert(RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
8411	}
8412
8413	/* If the GCM #DE exception handler didn't succeed or wasn't needed, raise #DE. */
8414	if (RT_FAILURE(rc))
8415	{
8416	hmR0SvmSetPendingXcptDE(pVCpu);
8417	rc = VINF_SUCCESS;
8418	}
8419
8420	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
8421	return rc;
8422	}
8423
8424
8425	/**
8426	* \#VMEXIT handler for undefined opcode (SVM_EXIT_XCPT_6).
8427	* Conditional \#VMEXIT.
8428	*/
8429	HMSVM_EXIT_DECL hmR0SvmExitXcptUD(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8430	{
8431	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8432	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
8433	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8434
8435	/* Paranoia; Ensure we cannot be called as a result of event delivery. */
8436	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
8437	Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
8438
8439	/** @todo if we accumulate more optional stuff here, we ought to combine the
8440	* reading of opcode bytes to avoid doing more than once. */
8441
8442	VBOXSTRICTRC rcStrict = VERR_SVM_UNEXPECTED_XCPT_EXIT;
8443	if (pVCpu->hm.s.fGIMTrapXcptUD)
8444	{
8445	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8446	uint8_t cbInstr = 0;
8447	rcStrict = GIMXcptUD(pVCpu, &pVCpu->cpum.GstCtx, NULL /* pDis */, &cbInstr);
8448	if (rcStrict == VINF_SUCCESS)
8449	{
8450	/* #UD #VMEXIT does not have valid NRIP information, manually advance RIP. See @bugref{7270#c170}. */
8451	hmR0SvmAdvanceRip(pVCpu, cbInstr);
8452	rcStrict = VINF_SUCCESS;
8453	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
8454	}
8455	else if (rcStrict == VINF_GIM_HYPERCALL_CONTINUING)
8456	rcStrict = VINF_SUCCESS;
8457	else if (rcStrict == VINF_GIM_R3_HYPERCALL)
8458	rcStrict = VINF_GIM_R3_HYPERCALL;
8459	else
8460	{
8461	Assert(RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
8462	rcStrict = VERR_SVM_UNEXPECTED_XCPT_EXIT;
8463	}
8464	}
8465
8466	if (pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit)
8467	{
8468	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS
8469	\| CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_EFER);
8470	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
8471	{
8472	/* Ideally, IEM should just handle all these special #UD situations, but
8473	we don't quite trust things to behave optimially when doing that. So,
8474	for now we'll restrict ourselves to a handful of possible sysenter and
8475	sysexit encodings that we filter right here. */
8476	uint8_t abInstr[SVM_CTRL_GUEST_INSTR_BYTES_MAX];
8477	uint8_t cbInstr = pVmcb->ctrl.cbInstrFetched;
8478	uint32_t const uCpl = CPUMGetGuestCPL(pVCpu);
8479	uint8_t const cbMin = uCpl != 0 ? 2 : 1 + 2;
8480	RTGCPTR const GCPtrInstr = pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base;
8481	if (cbInstr < cbMin \|\| cbInstr > SVM_CTRL_GUEST_INSTR_BYTES_MAX)
8482	{
8483	cbInstr = cbMin;
8484	int rc2 = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, GCPtrInstr, cbInstr);
8485	AssertRCStmt(rc2, cbInstr = 0);
8486	}
8487	else
8488	memcpy(abInstr, pVmcb->ctrl.abInstr, cbInstr); /* unlikely */
8489	if ( cbInstr == 0 /* read error */
8490	\|\| (cbInstr >= 2 && abInstr[0] == 0x0f && abInstr[1] == 0x34) /* sysenter */
8491	\|\| ( uCpl == 0
8492	&& ( ( cbInstr >= 2 && abInstr[0] == 0x0f && abInstr[1] == 0x35) /* sysexit */
8493	\|\| ( cbInstr >= 3 && abInstr[1] == 0x0f && abInstr[2] == 0x35 /* rex.w sysexit */
8494	&& (abInstr[0] & (X86_OP_REX_W \| 0xf0)) == X86_OP_REX_W))))
8495	{
8496	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
8497	\| CPUMCTX_EXTRN_SREG_MASK /* without ES+DS+GS the app will #GP later - go figure */);
8498	Log6(("hmR0SvmExitXcptUD: sysenter/sysexit: %.*Rhxs at %#llx CPL=%u\n", cbInstr, abInstr, GCPtrInstr, uCpl));
8499	rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), GCPtrInstr, abInstr, cbInstr);
8500	Log6(("hmR0SvmExitXcptUD: sysenter/sysexit: rcStrict=%Rrc %04x:%08RX64 %08RX64 %04x:%08RX64\n",
8501	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags.u,
8502	pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp));
8503	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8504	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); /** @todo Lazy bird. */
8505	if (rcStrict == VINF_IEM_RAISED_XCPT)
8506	rcStrict = VINF_SUCCESS;
8507	return rcStrict;
8508	}
8509	Log6(("hmR0SvmExitXcptUD: not sysenter/sysexit: %.*Rhxs at %#llx CPL=%u\n", cbInstr, abInstr, GCPtrInstr, uCpl));
8510	}
8511	else
8512	Log6(("hmR0SvmExitXcptUD: not in long mode at %04x:%llx\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
8513	}
8514
8515	/* If the GIM #UD exception handler didn't succeed for some reason or wasn't needed, raise #UD. */
8516	if (RT_FAILURE(rcStrict))
8517	{
8518	hmR0SvmSetPendingXcptUD(pVCpu);
8519	rcStrict = VINF_SUCCESS;
8520	}
8521
8522	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8523	return rcStrict;
8524	}
8525
8526
8527	/**
8528	* \#VMEXIT handler for math-fault exceptions (SVM_EXIT_XCPT_16).
8529	* Conditional \#VMEXIT.
8530	*/
8531	HMSVM_EXIT_DECL hmR0SvmExitXcptMF(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8532	{
8533	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8534	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8535	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8536
8537	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8538	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8539
8540	/* Paranoia; Ensure we cannot be called as a result of event delivery. */
8541	Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
8542
8543	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8544
8545	if (!(pCtx->cr0 & X86_CR0_NE))
8546	{
8547	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8548	PDISSTATE pDis = &pVCpu->hmr0.s.svm.DisState;
8549	unsigned cbInstr;
8550	int rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbInstr);
8551	if (RT_SUCCESS(rc))
8552	{
8553	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
8554	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13 /* u8Irq /, 1 / u8Level /, 0 / uTagSrc */);
8555	if (RT_SUCCESS(rc))
8556	hmR0SvmAdvanceRip(pVCpu, cbInstr);
8557	}
8558	else
8559	Log4Func(("EMInterpretDisasCurrent returned %Rrc uOpCode=%#x\n", rc, pDis->pCurInstr->uOpcode));
8560	return rc;
8561	}
8562
8563	hmR0SvmSetPendingXcptMF(pVCpu);
8564	return VINF_SUCCESS;
8565	}
8566
8567
8568	/**
8569	* \#VMEXIT handler for debug exceptions (SVM_EXIT_XCPT_1). Conditional
8570	* \#VMEXIT.
8571	*/
8572	HMSVM_EXIT_DECL hmR0SvmExitXcptDB(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8573	{
8574	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8575	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8576	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8577	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8578
8579	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
8580	{
8581	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
8582	return VINF_EM_RAW_INJECT_TRPM_EVENT;
8583	}
8584
8585	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8586
8587	/*
8588	* This can be a fault-type #DB (instruction breakpoint) or a trap-type #DB (data
8589	* breakpoint). However, for both cases DR6 and DR7 are updated to what the exception
8590	* handler expects. See AMD spec. 15.12.2 "#DB (Debug)".
8591	*/
8592	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8593	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
8594	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8595	int rc = DBGFTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), pVmcb->guest.u64DR6, pVCpu->hm.s.fSingleInstruction);
8596	if (rc == VINF_EM_RAW_GUEST_TRAP)
8597	{
8598	Log5(("hmR0SvmExitXcptDB: DR6=%#RX64 -> guest trap\n", pVmcb->guest.u64DR6));
8599	if (CPUMIsHyperDebugStateActive(pVCpu))
8600	CPUMSetGuestDR6(pVCpu, CPUMGetGuestDR6(pVCpu) \| pVmcb->guest.u64DR6);
8601
8602	/* Reflect the exception back to the guest. */
8603	hmR0SvmSetPendingXcptDB(pVCpu);
8604	rc = VINF_SUCCESS;
8605	}
8606
8607	/*
8608	* Update DR6.
8609	*/
8610	if (CPUMIsHyperDebugStateActive(pVCpu))
8611	{
8612	Log5(("hmR0SvmExitXcptDB: DR6=%#RX64 -> %Rrc\n", pVmcb->guest.u64DR6, rc));
8613	pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
8614	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
8615	}
8616	else
8617	{
8618	AssertMsg(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc));
8619	Assert(!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu));
8620	}
8621
8622	return rc;
8623	}
8624
8625
8626	/**
8627	* \#VMEXIT handler for alignment check exceptions (SVM_EXIT_XCPT_17).
8628	* Conditional \#VMEXIT.
8629	*/
8630	HMSVM_EXIT_DECL hmR0SvmExitXcptAC(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8631	{
8632	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8633	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8634	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC);
8635
8636	SVMEVENT Event;
8637	Event.u = 0;
8638	Event.n.u1Valid = 1;
8639	Event.n.u3Type = SVM_EVENT_EXCEPTION;
8640	Event.n.u8Vector = X86_XCPT_AC;
8641	Event.n.u1ErrorCodeValid = 1;
8642	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8643	return VINF_SUCCESS;
8644	}
8645
8646
8647	/**
8648	* \#VMEXIT handler for breakpoint exceptions (SVM_EXIT_XCPT_3).
8649	* Conditional \#VMEXIT.
8650	*/
8651	HMSVM_EXIT_DECL hmR0SvmExitXcptBP(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8652	{
8653	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8654	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8655	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8656	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8657
8658	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8659	VBOXSTRICTRC rc = DBGFTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
8660	if (rc == VINF_EM_RAW_GUEST_TRAP)
8661	{
8662	SVMEVENT Event;
8663	Event.u = 0;
8664	Event.n.u1Valid = 1;
8665	Event.n.u3Type = SVM_EVENT_EXCEPTION;
8666	Event.n.u8Vector = X86_XCPT_BP;
8667	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8668	rc = VINF_SUCCESS;
8669	}
8670
8671	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_DBG_BREAKPOINT);
8672	return rc;
8673	}
8674
8675
8676	/**
8677	* Hacks its way around the lovely mesa driver's backdoor accesses.
8678	*
8679	* @sa hmR0VmxHandleMesaDrvGp
8680	*/
8681	static int hmR0SvmHandleMesaDrvGp(PVMCPUCC pVCpu, PCPUMCTX pCtx, PCSVMVMCB pVmcb)
8682	{
8683	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_GPRS_MASK);
8684	Log(("hmR0SvmHandleMesaDrvGp: at %04x:%08RX64 rcx=%RX64 rbx=%RX64\n",
8685	pVmcb->guest.CS.u16Sel, pVmcb->guest.u64RIP, pCtx->rcx, pCtx->rbx));
8686	RT_NOREF(pCtx, pVmcb);
8687
8688	/* For now we'll just skip the instruction. */
8689	hmR0SvmAdvanceRip(pVCpu, 1);
8690	return VINF_SUCCESS;
8691	}
8692
8693
8694	/**
8695	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
8696	* backdoor logging w/o checking what it is running inside.
8697	*
8698	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
8699	* backdoor port and magic numbers loaded in registers.
8700	*
8701	* @returns true if it is, false if it isn't.
8702	* @sa hmR0VmxIsMesaDrvGp
8703	*/
8704	DECLINLINE(bool) hmR0SvmIsMesaDrvGp(PVMCPUCC pVCpu, PCPUMCTX pCtx, PCSVMVMCB pVmcb)
8705	{
8706	/* Check magic and port. */
8707	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
8708	/Log8(("hmR0SvmIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->fExtrn & CPUMCTX_EXTRN_RAX ? pVmcb->guest.u64RAX : pCtx->rax, pCtx->rdx));/
8709	if (pCtx->dx != UINT32_C(0x5658))
8710	return false;
8711	if ((pCtx->fExtrn & CPUMCTX_EXTRN_RAX ? pVmcb->guest.u64RAX : pCtx->rax) != UINT32_C(0x564d5868))
8712	return false;
8713
8714	/* Check that it is #GP(0). */
8715	if (pVmcb->ctrl.u64ExitInfo1 != 0)
8716	return false;
8717
8718	/* Flat ring-3 CS. */
8719	/Log8(("hmR0SvmIsMesaDrvGp: u8CPL=%d base=%RX64\n", pVmcb->guest.u8CPL, pCtx->fExtrn & CPUMCTX_EXTRN_CS ? pVmcb->guest.CS.u64Base : pCtx->cs.u64Base));/
8720	if (pVmcb->guest.u8CPL != 3)
8721	return false;
8722	if ((pCtx->fExtrn & CPUMCTX_EXTRN_CS ? pVmcb->guest.CS.u64Base : pCtx->cs.u64Base) != 0)
8723	return false;
8724
8725	/* 0xed: IN eAX,dx */
8726	if (pVmcb->ctrl.cbInstrFetched < 1) /* unlikely, it turns out. */
8727	{
8728	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_GPRS_MASK
8729	\| CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_EFER);
8730	uint8_t abInstr[1];
8731	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
8732	/Log8(("hmR0SvmIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0])); /
8733	if (RT_FAILURE(rc))
8734	return false;
8735	if (abInstr[0] != 0xed)
8736	return false;
8737	}
8738	else
8739	{
8740	/Log8(("hmR0SvmIsMesaDrvGp: %#x\n", pVmcb->ctrl.abInstr));/
8741	if (pVmcb->ctrl.abInstr[0] != 0xed)
8742	return false;
8743	}
8744	return true;
8745	}
8746
8747
8748	/**
8749	* \#VMEXIT handler for general protection faults (SVM_EXIT_XCPT_BP).
8750	* Conditional \#VMEXIT.
8751	*/
8752	HMSVM_EXIT_DECL hmR0SvmExitXcptGP(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8753	{
8754	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8755	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8756	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8757
8758	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8759	Assert(pSvmTransient->u64ExitCode == pVmcb->ctrl.u64ExitCode);
8760
8761	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8762	if ( !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
8763	\|\| !hmR0SvmIsMesaDrvGp(pVCpu, pCtx, pVmcb))
8764	{
8765	SVMEVENT Event;
8766	Event.u = 0;
8767	Event.n.u1Valid = 1;
8768	Event.n.u3Type = SVM_EVENT_EXCEPTION;
8769	Event.n.u8Vector = X86_XCPT_GP;
8770	Event.n.u1ErrorCodeValid = 1;
8771	Event.n.u32ErrorCode = (uint32_t)pVmcb->ctrl.u64ExitInfo1;
8772	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8773	return VINF_SUCCESS;
8774	}
8775	return hmR0SvmHandleMesaDrvGp(pVCpu, pCtx, pVmcb);
8776	}
8777
8778
8779	/**
8780	* \#VMEXIT handler for generic exceptions. Conditional \#VMEXIT.
8781	*/
8782	HMSVM_EXIT_DECL hmR0SvmExitXcptGeneric(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8783	{
8784	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8785	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8786
8787	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8788	uint8_t const uVector = pVmcb->ctrl.u64ExitCode - SVM_EXIT_XCPT_0;
8789	uint32_t const uErrCode = pVmcb->ctrl.u64ExitInfo1;
8790	Assert(pSvmTransient->u64ExitCode == pVmcb->ctrl.u64ExitCode);
8791	Assert(uVector <= X86_XCPT_LAST);
8792	Log4Func(("uVector=%#x uErrCode=%u\n", uVector, uErrCode));
8793
8794	SVMEVENT Event;
8795	Event.u = 0;
8796	Event.n.u1Valid = 1;
8797	Event.n.u3Type = SVM_EVENT_EXCEPTION;
8798	Event.n.u8Vector = uVector;
8799	switch (uVector)
8800	{
8801	/* Shouldn't be here for reflecting #PFs (among other things, the fault address isn't passed along). */
8802	case X86_XCPT_PF: AssertMsgFailed(("hmR0SvmExitXcptGeneric: Unexpected exception")); return VERR_SVM_IPE_5;
8803	case X86_XCPT_DF:
8804	case X86_XCPT_TS:
8805	case X86_XCPT_NP:
8806	case X86_XCPT_SS:
8807	case X86_XCPT_GP:
8808	case X86_XCPT_AC:
8809	{
8810	Event.n.u1ErrorCodeValid = 1;
8811	Event.n.u32ErrorCode = uErrCode;
8812	break;
8813	}
8814	}
8815
8816	#ifdef VBOX_WITH_STATISTICS
8817	switch (uVector)
8818	{
8819	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE); break;
8820	case X86_XCPT_DB: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB); break;
8821	case X86_XCPT_BP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP); break;
8822	case X86_XCPT_OF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
8823	case X86_XCPT_BR: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBR); break;
8824	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD); break;
8825	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
8826	case X86_XCPT_DF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDF); break;
8827	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS); break;
8828	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP); break;
8829	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS); break;
8830	case X86_XCPT_GP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP); break;
8831	case X86_XCPT_PF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF); break;
8832	case X86_XCPT_MF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF); break;
8833	case X86_XCPT_AC: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC); break;
8834	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF); break;
8835	default:
8836	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
8837	break;
8838	}
8839	#endif
8840
8841	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8842	return VINF_SUCCESS;
8843	}
8844
8845
8846	/**
8847	* \#VMEXIT handler for software interrupt (INTn). Conditional \#VMEXIT (debug).
8848	*/
8849	HMSVM_EXIT_DECL hmR0SvmExitSwInt(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8850	{
8851	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8852	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8853
8854	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8855	SVMEVENT Event;
8856	Event.u = 0;
8857	Event.n.u1Valid = 1;
8858	Event.n.u3Type = SVM_EVENT_SOFTWARE_INT;
8859	Event.n.u8Vector = pVmcb->ctrl.u64ExitInfo1 & 0xff;
8860	Log4Func(("uVector=%#x\n", Event.n.u8Vector));
8861	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8862	return VINF_SUCCESS;
8863	}
8864
8865
8866	/**
8867	* Generic exit handler that interprets the current instruction
8868	*
8869	* Useful exit that only gets triggered by dtrace and the debugger. Caller does
8870	* the exit logging, and this function does the rest.
8871	*/
8872	static VBOXSTRICTRC hmR0SvmExitInterpretInstruction(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient,
8873	uint64_t fExtraImport, uint64_t fHmChanged)
8874	{
8875	#if 1
8876	RT_NOREF(pSvmTransient);
8877	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fExtraImport);
8878	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
8879	if (rcStrict == VINF_SUCCESS)
8880	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, fHmChanged \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RIP);
8881	else
8882	{
8883	Log4Func(("IEMExecOne -> %Rrc\n", VBOXSTRICTRC_VAL(rcStrict) ));
8884	if (rcStrict == VINF_IEM_RAISED_XCPT)
8885	{
8886	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK \| fHmChanged);
8887	rcStrict = VINF_SUCCESS;
8888	}
8889	else
8890	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, fHmChanged);
8891	}
8892	return rcStrict;
8893	#else
8894	RT_NOREF(pVCpu, pSvmTransient, fExtraImport, fHmChanged);
8895	return VINF_EM_RAW_EMULATE_INSTR;
8896	#endif
8897	}
8898
8899
8900	/**
8901	* \#VMEXIT handler for STR. Conditional \#VMEXIT (debug).
8902	*/
8903	HMSVM_EXIT_DECL hmR0SvmExitTrRead(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8904	{
8905	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8906	Log4Func(("%04x:%08RX64\n", pSvmTransient->pVmcb->guest.CS.u16Sel, pSvmTransient->pVmcb->guest.u64RIP));
8907	return hmR0SvmExitInterpretInstruction(pVCpu, pSvmTransient, CPUMCTX_EXTRN_TR, 0);
8908	}
8909
8910
8911	/**
8912	* \#VMEXIT handler for LTR. Conditional \#VMEXIT (OS/2 TLB workaround, debug).
8913	*/
8914	HMSVM_EXIT_DECL hmR0SvmExitTrWrite(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8915	{
8916	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8917
8918	/* Workaround for lack of TLB flushing in OS/2 when returning to protected
8919	mode after a real mode call (like a BIOS call). See ticketref:20625
8920	comment 14. */
8921	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8922	if (pVM->hm.s.fMissingOS2TlbFlushWorkaround)
8923	{
8924	Log4Func(("%04x:%08RX64 TLB flush\n", pSvmTransient->pVmcb->guest.CS.u16Sel, pSvmTransient->pVmcb->guest.u64RIP));
8925	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
8926	}
8927	else
8928	Log4Func(("%04x:%08RX64\n", pSvmTransient->pVmcb->guest.CS.u16Sel, pSvmTransient->pVmcb->guest.u64RIP));
8929
8930	return hmR0SvmExitInterpretInstruction(pVCpu, pSvmTransient, CPUMCTX_EXTRN_TR \| CPUMCTX_EXTRN_GDTR, HM_CHANGED_GUEST_TR);
8931	}
8932
8933
8934	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
8935	/**
8936	* \#VMEXIT handler for CLGI (SVM_EXIT_CLGI). Conditional \#VMEXIT.
8937	*/
8938	HMSVM_EXIT_DECL hmR0SvmExitClgi(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8939	{
8940	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8941
8942	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8943	Assert(pVmcb);
8944	Assert(!pVmcb->ctrl.IntCtrl.n.u1VGifEnable);
8945
8946	VBOXSTRICTRC rcStrict;
8947	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8948	uint64_t const fImport = CPUMCTX_EXTRN_HWVIRT;
8949	if (fSupportsNextRipSave)
8950	{
8951	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| fImport);
8952	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8953	rcStrict = IEMExecDecodedClgi(pVCpu, cbInstr);
8954	}
8955	else
8956	{
8957	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fImport);
8958	rcStrict = IEMExecOne(pVCpu);
8959	}
8960
8961	if (rcStrict == VINF_SUCCESS)
8962	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
8963	else if (rcStrict == VINF_IEM_RAISED_XCPT)
8964	{
8965	rcStrict = VINF_SUCCESS;
8966	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
8967	}
8968	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
8969	return rcStrict;
8970	}
8971
8972
8973	/**
8974	* \#VMEXIT handler for STGI (SVM_EXIT_STGI). Conditional \#VMEXIT.
8975	*/
8976	HMSVM_EXIT_DECL hmR0SvmExitStgi(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8977	{
8978	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8979
8980	/*
8981	* When VGIF is not used we always intercept STGI instructions. When VGIF is used,
8982	* we only intercept STGI when events are pending for GIF to become 1.
8983	*/
8984	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8985	if (pVmcb->ctrl.IntCtrl.n.u1VGifEnable)
8986	hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_STGI);
8987
8988	VBOXSTRICTRC rcStrict;
8989	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8990	uint64_t const fImport = CPUMCTX_EXTRN_HWVIRT;
8991	if (fSupportsNextRipSave)
8992	{
8993	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| fImport);
8994	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8995	rcStrict = IEMExecDecodedStgi(pVCpu, cbInstr);
8996	}
8997	else
8998	{
8999	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fImport);
9000	rcStrict = IEMExecOne(pVCpu);
9001	}
9002
9003	if (rcStrict == VINF_SUCCESS)
9004	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
9005	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9006	{
9007	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9008	rcStrict = VINF_SUCCESS;
9009	}
9010	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
9011	return rcStrict;
9012	}
9013
9014
9015	/**
9016	* \#VMEXIT handler for VMLOAD (SVM_EXIT_VMLOAD). Conditional \#VMEXIT.
9017	*/
9018	HMSVM_EXIT_DECL hmR0SvmExitVmload(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9019	{
9020	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9021
9022	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
9023	Assert(pVmcb);
9024	Assert(!pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload);
9025
9026	VBOXSTRICTRC rcStrict;
9027	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
9028	uint64_t const fImport = CPUMCTX_EXTRN_FS \| CPUMCTX_EXTRN_GS \| CPUMCTX_EXTRN_KERNEL_GS_BASE
9029	\| CPUMCTX_EXTRN_TR \| CPUMCTX_EXTRN_LDTR \| CPUMCTX_EXTRN_SYSCALL_MSRS
9030	\| CPUMCTX_EXTRN_SYSENTER_MSRS;
9031	if (fSupportsNextRipSave)
9032	{
9033	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| fImport);
9034	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
9035	rcStrict = IEMExecDecodedVmload(pVCpu, cbInstr);
9036	}
9037	else
9038	{
9039	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fImport);
9040	rcStrict = IEMExecOne(pVCpu);
9041	}
9042
9043	if (rcStrict == VINF_SUCCESS)
9044	{
9045	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS \| HM_CHANGED_GUEST_GS
9046	\| HM_CHANGED_GUEST_TR \| HM_CHANGED_GUEST_LDTR
9047	\| HM_CHANGED_GUEST_KERNEL_GS_BASE \| HM_CHANGED_GUEST_SYSCALL_MSRS
9048	\| HM_CHANGED_GUEST_SYSENTER_MSR_MASK);
9049	}
9050	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9051	{
9052	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9053	rcStrict = VINF_SUCCESS;
9054	}
9055	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
9056	return rcStrict;
9057	}
9058
9059
9060	/**
9061	* \#VMEXIT handler for VMSAVE (SVM_EXIT_VMSAVE). Conditional \#VMEXIT.
9062	*/
9063	HMSVM_EXIT_DECL hmR0SvmExitVmsave(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9064	{
9065	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9066
9067	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
9068	Assert(!pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload);
9069
9070	VBOXSTRICTRC rcStrict;
9071	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
9072	if (fSupportsNextRipSave)
9073	{
9074	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
9075	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
9076	rcStrict = IEMExecDecodedVmsave(pVCpu, cbInstr);
9077	}
9078	else
9079	{
9080	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
9081	rcStrict = IEMExecOne(pVCpu);
9082	}
9083
9084	if (rcStrict == VINF_IEM_RAISED_XCPT)
9085	{
9086	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9087	rcStrict = VINF_SUCCESS;
9088	}
9089	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
9090	return rcStrict;
9091	}
9092
9093
9094	/**
9095	* \#VMEXIT handler for INVLPGA (SVM_EXIT_INVLPGA). Conditional \#VMEXIT.
9096	*/
9097	HMSVM_EXIT_DECL hmR0SvmExitInvlpga(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9098	{
9099	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9100
9101	VBOXSTRICTRC rcStrict;
9102	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
9103	if (fSupportsNextRipSave)
9104	{
9105	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
9106	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
9107	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
9108	rcStrict = IEMExecDecodedInvlpga(pVCpu, cbInstr);
9109	}
9110	else
9111	{
9112	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
9113	rcStrict = IEMExecOne(pVCpu);
9114	}
9115
9116	if (rcStrict == VINF_IEM_RAISED_XCPT)
9117	{
9118	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9119	rcStrict = VINF_SUCCESS;
9120	}
9121	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
9122	return rcStrict;
9123	}
9124
9125
9126	/**
9127	* \#VMEXIT handler for STGI (SVM_EXIT_VMRUN). Conditional \#VMEXIT.
9128	*/
9129	HMSVM_EXIT_DECL hmR0SvmExitVmrun(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9130	{
9131	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9132	/* We shall import the entire state here, just in case we enter and continue execution of
9133	the nested-guest with hardware-assisted SVM in ring-0, we would be switching VMCBs and
9134	could lose lose part of CPU state. */
9135	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
9136
9137	VBOXSTRICTRC rcStrict;
9138	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
9139	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
9140	if (fSupportsNextRipSave)
9141	{
9142	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
9143	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
9144	rcStrict = IEMExecDecodedVmrun(pVCpu, cbInstr);
9145	}
9146	else
9147	{
9148	/* We use IEMExecOneBypassEx() here as it supresses attempt to continue emulating any
9149	instruction(s) when interrupt inhibition is set as part of emulating the VMRUN
9150	instruction itself, see @bugref{7243#c126} */
9151	rcStrict = IEMExecOneBypassEx(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), NULL /* pcbWritten */);
9152	}
9153	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
9154
9155	if (rcStrict == VINF_SUCCESS)
9156	{
9157	rcStrict = VINF_SVM_VMRUN;
9158	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_SVM_VMRUN_MASK);
9159	}
9160	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9161	{
9162	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9163	rcStrict = VINF_SUCCESS;
9164	}
9165	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
9166	return rcStrict;
9167	}
9168
9169
9170	/**
9171	* Nested-guest \#VMEXIT handler for debug exceptions (SVM_EXIT_XCPT_1).
9172	* Unconditional \#VMEXIT.
9173	*/
9174	HMSVM_EXIT_DECL hmR0SvmNestedExitXcptDB(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9175	{
9176	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9177	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
9178
9179	if (pVCpu->hm.s.Event.fPending)
9180	{
9181	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
9182	return VINF_EM_RAW_INJECT_TRPM_EVENT;
9183	}
9184
9185	hmR0SvmSetPendingXcptDB(pVCpu);
9186	return VINF_SUCCESS;
9187	}
9188
9189
9190	/**
9191	* Nested-guest \#VMEXIT handler for breakpoint exceptions (SVM_EXIT_XCPT_3).
9192	* Conditional \#VMEXIT.
9193	*/
9194	HMSVM_EXIT_DECL hmR0SvmNestedExitXcptBP(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9195	{
9196	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9197	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
9198
9199	SVMEVENT Event;
9200	Event.u = 0;
9201	Event.n.u1Valid = 1;
9202	Event.n.u3Type = SVM_EVENT_EXCEPTION;
9203	Event.n.u8Vector = X86_XCPT_BP;
9204	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
9205	return VINF_SUCCESS;
9206	}
9207	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
9208
9209	/** @} */
9210

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMSVMR0.cpp@ 95248

Download in other formats: