HMSVMR0.cpp@ 95495

Last change on this file since 95495 was 95495, checked in by vboxsync, 2 years ago
VMM/HMSVMR0: updated CR4 write intercept comments with AVX/xcr0.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 382.0 KB

Line
1	/* $Id: HMSVMR0.cpp 95495 2022-07-04 08:18:19Z vboxsync $ */
2	/** @file
3	* HM SVM (AMD-V) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2013-2022 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/asm-amd64-x86.h>
25	#include <iprt/thread.h>
26
27	#include <VBox/vmm/pdmapi.h>
28	#include <VBox/vmm/dbgf.h>
29	#include <VBox/vmm/iem.h>
30	#include <VBox/vmm/iom.h>
31	#include <VBox/vmm/tm.h>
32	#include <VBox/vmm/em.h>
33	#include <VBox/vmm/gcm.h>
34	#include <VBox/vmm/gim.h>
35	#include <VBox/vmm/apic.h>
36	#include "HMInternal.h"
37	#include <VBox/vmm/vmcc.h>
38	#include <VBox/err.h>
39	#include "HMSVMR0.h"
40	#include "dtrace/VBoxVMM.h"
41
42	#ifdef DEBUG_ramshankar
43	# define HMSVM_SYNC_FULL_GUEST_STATE
44	# define HMSVM_ALWAYS_TRAP_ALL_XCPTS
45	# define HMSVM_ALWAYS_TRAP_PF
46	# define HMSVM_ALWAYS_TRAP_TASK_SWITCH
47	#endif
48
49
50	/*********************************************************************************************************************************
51	* Defined Constants And Macros *
52	*********************************************************************************************************************************/
53	#ifdef VBOX_WITH_STATISTICS
54	# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
55	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll); \
56	if ((u64ExitCode) == SVM_EXIT_NPF) \
57	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf); \
58	else \
59	STAM_COUNTER_INC(&pVCpu->hm.s.aStatExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
60	} while (0)
61
62	# define HMSVM_DEBUG_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
63	STAM_COUNTER_INC(&pVCpu->hm.s.StatDebugExitAll); \
64	if ((u64ExitCode) == SVM_EXIT_NPF) \
65	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf); \
66	else \
67	STAM_COUNTER_INC(&pVCpu->hm.s.aStatExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
68	} while (0)
69
70	# define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
71	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitAll); \
72	if ((u64ExitCode) == SVM_EXIT_NPF) \
73	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitReasonNpf); \
74	else \
75	STAM_COUNTER_INC(&pVCpu->hm.s.aStatNestedExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
76	} while (0)
77	#else
78	# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
79	# define HMSVM_DEBUG_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
80	# define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
81	#endif /* !VBOX_WITH_STATISTICS */
82
83	/** If we decide to use a function table approach this can be useful to
84	* switch to a "static DECLCALLBACK(int)". */
85	#define HMSVM_EXIT_DECL static VBOXSTRICTRC
86
87	/**
88	* Subset of the guest-CPU state that is kept by SVM R0 code while executing the
89	* guest using hardware-assisted SVM.
90	*
91	* This excludes state like TSC AUX, GPRs (other than RSP, RAX) which are always
92	* are swapped and restored across the world-switch and also registers like
93	* EFER, PAT MSR etc. which cannot be modified by the guest without causing a
94	* \#VMEXIT.
95	*/
96	#define HMSVM_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
97	\| CPUMCTX_EXTRN_RFLAGS \
98	\| CPUMCTX_EXTRN_RAX \
99	\| CPUMCTX_EXTRN_RSP \
100	\| CPUMCTX_EXTRN_SREG_MASK \
101	\| CPUMCTX_EXTRN_CR0 \
102	\| CPUMCTX_EXTRN_CR2 \
103	\| CPUMCTX_EXTRN_CR3 \
104	\| CPUMCTX_EXTRN_TABLE_MASK \
105	\| CPUMCTX_EXTRN_DR6 \
106	\| CPUMCTX_EXTRN_DR7 \
107	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
108	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
109	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
110	\| CPUMCTX_EXTRN_HWVIRT \
111	\| CPUMCTX_EXTRN_INHIBIT_INT \
112	\| CPUMCTX_EXTRN_HM_SVM_MASK)
113
114	/**
115	* Subset of the guest-CPU state that is shared between the guest and host.
116	*/
117	#define HMSVM_CPUMCTX_SHARED_STATE CPUMCTX_EXTRN_DR_MASK
118
119	/** Macro for importing guest state from the VMCB back into CPUMCTX. */
120	#define HMSVM_CPUMCTX_IMPORT_STATE(a_pVCpu, a_fWhat) \
121	do { \
122	if ((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fWhat)) \
123	hmR0SvmImportGuestState((a_pVCpu), (a_fWhat)); \
124	} while (0)
125
126	/** Assert that the required state bits are fetched. */
127	#define HMSVM_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
128	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
129	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
130
131	/** Assert that preemption is disabled or covered by thread-context hooks. */
132	#define HMSVM_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
133	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
134
135	/** Assert that we haven't migrated CPUs when thread-context hooks are not
136	* used. */
137	#define HMSVM_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
138	\|\| (a_pVCpu)->hmr0.s.idEnteredCpu == RTMpCpuId(), \
139	("Illegal migration! Entered on CPU %u Current %u\n", \
140	(a_pVCpu)->hmr0.s.idEnteredCpu, RTMpCpuId()));
141
142	/** Assert that we're not executing a nested-guest. */
143	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
144	# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) Assert(!CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
145	#else
146	# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
147	#endif
148
149	/** Assert that we're executing a nested-guest. */
150	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
151	# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) Assert(CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
152	#else
153	# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
154	#endif
155
156	/** Macro for checking and returning from the using function for
157	* \#VMEXIT intercepts that maybe caused during delivering of another
158	* event in the guest. */
159	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
160	# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(a_pVCpu, a_pSvmTransient) \
161	do \
162	{ \
163	int rc = hmR0SvmCheckExitDueToEventDelivery((a_pVCpu), (a_pSvmTransient)); \
164	if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
165	else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
166	else if ( rc == VINF_EM_RESET \
167	&& CPUMIsGuestSvmCtrlInterceptSet((a_pVCpu), &(a_pVCpu)->cpum.GstCtx, SVM_CTRL_INTERCEPT_SHUTDOWN)) \
168	{ \
169	HMSVM_CPUMCTX_IMPORT_STATE((a_pVCpu), HMSVM_CPUMCTX_EXTRN_ALL); \
170	return IEMExecSvmVmexit((a_pVCpu), SVM_EXIT_SHUTDOWN, 0, 0); \
171	} \
172	else \
173	return rc; \
174	} while (0)
175	#else
176	# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(a_pVCpu, a_pSvmTransient) \
177	do \
178	{ \
179	int rc = hmR0SvmCheckExitDueToEventDelivery((a_pVCpu), (a_pSvmTransient)); \
180	if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
181	else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
182	else \
183	return rc; \
184	} while (0)
185	#endif
186
187	/** Macro for upgrading a @a a_rc to VINF_EM_DBG_STEPPED after emulating an
188	* instruction that exited. */
189	#define HMSVM_CHECK_SINGLE_STEP(a_pVCpu, a_rc) \
190	do { \
191	if ((a_pVCpu)->hm.s.fSingleInstruction && (a_rc) == VINF_SUCCESS) \
192	(a_rc) = VINF_EM_DBG_STEPPED; \
193	} while (0)
194
195	/** Validate segment descriptor granularity bit. */
196	#ifdef VBOX_STRICT
197	# define HMSVM_ASSERT_SEG_GRANULARITY(a_pCtx, reg) \
198	AssertMsg( !(a_pCtx)->reg.Attr.n.u1Present \
199	\|\| ( (a_pCtx)->reg.Attr.n.u1Granularity \
200	? ((a_pCtx)->reg.u32Limit & 0xfff) == 0xfff \
201	: (a_pCtx)->reg.u32Limit <= UINT32_C(0xfffff)), \
202	("Invalid Segment Attributes Limit=%#RX32 Attr=%#RX32 Base=%#RX64\n", (a_pCtx)->reg.u32Limit, \
203	(a_pCtx)->reg.Attr.u, (a_pCtx)->reg.u64Base))
204	#else
205	# define HMSVM_ASSERT_SEG_GRANULARITY(a_pCtx, reg) do { } while (0)
206	#endif
207
208	/**
209	* Exception bitmap mask for all contributory exceptions.
210	*
211	* Page fault is deliberately excluded here as it's conditional as to whether
212	* it's contributory or benign. Page faults are handled separately.
213	*/
214	#define HMSVM_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
215	\| RT_BIT(X86_XCPT_DE))
216
217	/**
218	* Mandatory/unconditional guest control intercepts.
219	*
220	* SMIs can and do happen in normal operation. We need not intercept them
221	* while executing the guest (or nested-guest).
222	*/
223	#define HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS ( SVM_CTRL_INTERCEPT_INTR \
224	\| SVM_CTRL_INTERCEPT_NMI \
225	\| SVM_CTRL_INTERCEPT_INIT \
226	\| SVM_CTRL_INTERCEPT_RDPMC \
227	\| SVM_CTRL_INTERCEPT_CPUID \
228	\| SVM_CTRL_INTERCEPT_RSM \
229	\| SVM_CTRL_INTERCEPT_HLT \
230	\| SVM_CTRL_INTERCEPT_IOIO_PROT \
231	\| SVM_CTRL_INTERCEPT_MSR_PROT \
232	\| SVM_CTRL_INTERCEPT_INVLPGA \
233	\| SVM_CTRL_INTERCEPT_SHUTDOWN \
234	\| SVM_CTRL_INTERCEPT_FERR_FREEZE \
235	\| SVM_CTRL_INTERCEPT_VMRUN \
236	\| SVM_CTRL_INTERCEPT_SKINIT \
237	\| SVM_CTRL_INTERCEPT_WBINVD \
238	\| SVM_CTRL_INTERCEPT_MONITOR \
239	\| SVM_CTRL_INTERCEPT_MWAIT \
240	\| SVM_CTRL_INTERCEPT_CR0_SEL_WRITE \
241	\| SVM_CTRL_INTERCEPT_XSETBV)
242
243	/** @name VMCB Clean Bits.
244	*
245	* These flags are used for VMCB-state caching. A set VMCB Clean bit indicates
246	* AMD-V doesn't need to reload the corresponding value(s) from the VMCB in
247	* memory.
248	*
249	* @{ */
250	/** All intercepts vectors, TSC offset, PAUSE filter counter. */
251	#define HMSVM_VMCB_CLEAN_INTERCEPTS RT_BIT(0)
252	/** I/O permission bitmap, MSR permission bitmap. */
253	#define HMSVM_VMCB_CLEAN_IOPM_MSRPM RT_BIT(1)
254	/** ASID. */
255	#define HMSVM_VMCB_CLEAN_ASID RT_BIT(2)
256	/** TRP: V_TPR, V_IRQ, V_INTR_PRIO, V_IGN_TPR, V_INTR_MASKING,
257	V_INTR_VECTOR. */
258	#define HMSVM_VMCB_CLEAN_INT_CTRL RT_BIT(3)
259	/** Nested Paging: Nested CR3 (nCR3), PAT. */
260	#define HMSVM_VMCB_CLEAN_NP RT_BIT(4)
261	/** Control registers (CR0, CR3, CR4, EFER). */
262	#define HMSVM_VMCB_CLEAN_CRX_EFER RT_BIT(5)
263	/** Debug registers (DR6, DR7). */
264	#define HMSVM_VMCB_CLEAN_DRX RT_BIT(6)
265	/** GDT, IDT limit and base. */
266	#define HMSVM_VMCB_CLEAN_DT RT_BIT(7)
267	/** Segment register: CS, SS, DS, ES limit and base. */
268	#define HMSVM_VMCB_CLEAN_SEG RT_BIT(8)
269	/** CR2.*/
270	#define HMSVM_VMCB_CLEAN_CR2 RT_BIT(9)
271	/** Last-branch record (DbgCtlMsr, br_from, br_to, lastint_from, lastint_to) */
272	#define HMSVM_VMCB_CLEAN_LBR RT_BIT(10)
273	/** AVIC (AVIC APIC_BAR; AVIC APIC_BACKING_PAGE, AVIC
274	PHYSICAL_TABLE and AVIC LOGICAL_TABLE Pointers). */
275	#define HMSVM_VMCB_CLEAN_AVIC RT_BIT(11)
276	/** Mask of all valid VMCB Clean bits. */
277	#define HMSVM_VMCB_CLEAN_ALL ( HMSVM_VMCB_CLEAN_INTERCEPTS \
278	\| HMSVM_VMCB_CLEAN_IOPM_MSRPM \
279	\| HMSVM_VMCB_CLEAN_ASID \
280	\| HMSVM_VMCB_CLEAN_INT_CTRL \
281	\| HMSVM_VMCB_CLEAN_NP \
282	\| HMSVM_VMCB_CLEAN_CRX_EFER \
283	\| HMSVM_VMCB_CLEAN_DRX \
284	\| HMSVM_VMCB_CLEAN_DT \
285	\| HMSVM_VMCB_CLEAN_SEG \
286	\| HMSVM_VMCB_CLEAN_CR2 \
287	\| HMSVM_VMCB_CLEAN_LBR \
288	\| HMSVM_VMCB_CLEAN_AVIC)
289	/** @} */
290
291	/**
292	* MSRPM (MSR permission bitmap) read permissions (for guest RDMSR).
293	*/
294	typedef enum SVMMSREXITREAD
295	{
296	/** Reading this MSR causes a \#VMEXIT. */
297	SVMMSREXIT_INTERCEPT_READ = 0xb,
298	/** Reading this MSR does not cause a \#VMEXIT. */
299	SVMMSREXIT_PASSTHRU_READ
300	} SVMMSREXITREAD;
301
302	/**
303	* MSRPM (MSR permission bitmap) write permissions (for guest WRMSR).
304	*/
305	typedef enum SVMMSREXITWRITE
306	{
307	/** Writing to this MSR causes a \#VMEXIT. */
308	SVMMSREXIT_INTERCEPT_WRITE = 0xd,
309	/** Writing to this MSR does not cause a \#VMEXIT. */
310	SVMMSREXIT_PASSTHRU_WRITE
311	} SVMMSREXITWRITE;
312
313	/**
314	* SVM \#VMEXIT handler.
315	*
316	* @returns Strict VBox status code.
317	* @param pVCpu The cross context virtual CPU structure.
318	* @param pSvmTransient Pointer to the SVM-transient structure.
319	*/
320	typedef VBOXSTRICTRC FNSVMEXITHANDLER(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
321
322
323	/*********************************************************************************************************************************
324	* Internal Functions *
325	*********************************************************************************************************************************/
326	static void hmR0SvmPendingEventToTrpmTrap(PVMCPUCC pVCpu);
327	static void hmR0SvmLeave(PVMCPUCC pVCpu, bool fImportState);
328
329
330	/** @name \#VMEXIT handlers.
331	* @{
332	*/
333	static FNSVMEXITHANDLER hmR0SvmExitIntr;
334	static FNSVMEXITHANDLER hmR0SvmExitWbinvd;
335	static FNSVMEXITHANDLER hmR0SvmExitInvd;
336	static FNSVMEXITHANDLER hmR0SvmExitCpuid;
337	static FNSVMEXITHANDLER hmR0SvmExitRdtsc;
338	static FNSVMEXITHANDLER hmR0SvmExitRdtscp;
339	static FNSVMEXITHANDLER hmR0SvmExitRdpmc;
340	static FNSVMEXITHANDLER hmR0SvmExitInvlpg;
341	static FNSVMEXITHANDLER hmR0SvmExitHlt;
342	static FNSVMEXITHANDLER hmR0SvmExitMonitor;
343	static FNSVMEXITHANDLER hmR0SvmExitMwait;
344	static FNSVMEXITHANDLER hmR0SvmExitShutdown;
345	static FNSVMEXITHANDLER hmR0SvmExitUnexpected;
346	static FNSVMEXITHANDLER hmR0SvmExitReadCRx;
347	static FNSVMEXITHANDLER hmR0SvmExitWriteCRx;
348	static FNSVMEXITHANDLER hmR0SvmExitMsr;
349	static FNSVMEXITHANDLER hmR0SvmExitReadDRx;
350	static FNSVMEXITHANDLER hmR0SvmExitWriteDRx;
351	static FNSVMEXITHANDLER hmR0SvmExitXsetbv;
352	static FNSVMEXITHANDLER hmR0SvmExitIOInstr;
353	static FNSVMEXITHANDLER hmR0SvmExitNestedPF;
354	static FNSVMEXITHANDLER hmR0SvmExitVIntr;
355	static FNSVMEXITHANDLER hmR0SvmExitTaskSwitch;
356	static FNSVMEXITHANDLER hmR0SvmExitVmmCall;
357	static FNSVMEXITHANDLER hmR0SvmExitPause;
358	static FNSVMEXITHANDLER hmR0SvmExitFerrFreeze;
359	static FNSVMEXITHANDLER hmR0SvmExitIret;
360	static FNSVMEXITHANDLER hmR0SvmExitXcptDE;
361	static FNSVMEXITHANDLER hmR0SvmExitXcptPF;
362	static FNSVMEXITHANDLER hmR0SvmExitXcptUD;
363	static FNSVMEXITHANDLER hmR0SvmExitXcptMF;
364	static FNSVMEXITHANDLER hmR0SvmExitXcptDB;
365	static FNSVMEXITHANDLER hmR0SvmExitXcptAC;
366	static FNSVMEXITHANDLER hmR0SvmExitXcptBP;
367	static FNSVMEXITHANDLER hmR0SvmExitXcptGP;
368	static FNSVMEXITHANDLER hmR0SvmExitXcptGeneric;
369	static FNSVMEXITHANDLER hmR0SvmExitSwInt;
370	static FNSVMEXITHANDLER hmR0SvmExitTrRead;
371	static FNSVMEXITHANDLER hmR0SvmExitTrWrite;
372	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
373	static FNSVMEXITHANDLER hmR0SvmExitClgi;
374	static FNSVMEXITHANDLER hmR0SvmExitStgi;
375	static FNSVMEXITHANDLER hmR0SvmExitVmload;
376	static FNSVMEXITHANDLER hmR0SvmExitVmsave;
377	static FNSVMEXITHANDLER hmR0SvmExitInvlpga;
378	static FNSVMEXITHANDLER hmR0SvmExitVmrun;
379	static FNSVMEXITHANDLER hmR0SvmNestedExitXcptDB;
380	static FNSVMEXITHANDLER hmR0SvmNestedExitXcptBP;
381	#endif
382	/** @} */
383
384	static VBOXSTRICTRC hmR0SvmHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
385	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
386	static VBOXSTRICTRC hmR0SvmHandleExitNested(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
387	#endif
388	static VBOXSTRICTRC hmR0SvmRunGuestCodeDebug(PVMCPUCC pVCpu, uint32_t *pcLoops);
389
390
391	/*********************************************************************************************************************************
392	* Global Variables *
393	*********************************************************************************************************************************/
394	/** Ring-0 memory object for the IO bitmap. */
395	static RTR0MEMOBJ g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
396	/** Physical address of the IO bitmap. */
397	static RTHCPHYS g_HCPhysIOBitmap;
398	/** Pointer to the IO bitmap. */
399	static R0PTRTYPE(void *) g_pvIOBitmap;
400
401	#ifdef VBOX_STRICT
402	# define HMSVM_LOG_RBP_RSP RT_BIT_32(0)
403	# define HMSVM_LOG_CR_REGS RT_BIT_32(1)
404	# define HMSVM_LOG_CS RT_BIT_32(2)
405	# define HMSVM_LOG_SS RT_BIT_32(3)
406	# define HMSVM_LOG_FS RT_BIT_32(4)
407	# define HMSVM_LOG_GS RT_BIT_32(5)
408	# define HMSVM_LOG_LBR RT_BIT_32(6)
409	# define HMSVM_LOG_ALL ( HMSVM_LOG_RBP_RSP \
410	\| HMSVM_LOG_CR_REGS \
411	\| HMSVM_LOG_CS \
412	\| HMSVM_LOG_SS \
413	\| HMSVM_LOG_FS \
414	\| HMSVM_LOG_GS \
415	\| HMSVM_LOG_LBR)
416
417	/**
418	* Dumps virtual CPU state and additional info. to the logger for diagnostics.
419	*
420	* @param pVCpu The cross context virtual CPU structure.
421	* @param pVmcb Pointer to the VM control block.
422	* @param pszPrefix Log prefix.
423	* @param fFlags Log flags, see HMSVM_LOG_XXX.
424	* @param uVerbose The verbosity level, currently unused.
425	*/
426	static void hmR0SvmLogState(PVMCPUCC pVCpu, PCSVMVMCB pVmcb, const char *pszPrefix, uint32_t fFlags, uint8_t uVerbose)
427	{
428	RT_NOREF2(pVCpu, uVerbose);
429	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
430
431	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
432	Log4(("%s: cs:rip=%04x:%RX64 efl=%#RX64\n", pszPrefix, pCtx->cs.Sel, pCtx->rip, pCtx->rflags.u));
433
434	if (fFlags & HMSVM_LOG_RBP_RSP)
435	{
436	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RBP);
437	Log4(("%s: rsp=%#RX64 rbp=%#RX64\n", pszPrefix, pCtx->rsp, pCtx->rbp));
438	}
439
440	if (fFlags & HMSVM_LOG_CR_REGS)
441	{
442	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4);
443	Log4(("%s: cr0=%#RX64 cr3=%#RX64 cr4=%#RX64\n", pszPrefix, pCtx->cr0, pCtx->cr3, pCtx->cr4));
444	}
445
446	if (fFlags & HMSVM_LOG_CS)
447	{
448	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
449	Log4(("%s: cs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->cs.Sel, pCtx->cs.u64Base,
450	pCtx->cs.u32Limit, pCtx->cs.Attr.u));
451	}
452	if (fFlags & HMSVM_LOG_SS)
453	{
454	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
455	Log4(("%s: ss={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->ss.Sel, pCtx->ss.u64Base,
456	pCtx->ss.u32Limit, pCtx->ss.Attr.u));
457	}
458	if (fFlags & HMSVM_LOG_FS)
459	{
460	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
461	Log4(("%s: fs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->fs.Sel, pCtx->fs.u64Base,
462	pCtx->fs.u32Limit, pCtx->fs.Attr.u));
463	}
464	if (fFlags & HMSVM_LOG_GS)
465	{
466	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
467	Log4(("%s: gs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->gs.Sel, pCtx->gs.u64Base,
468	pCtx->gs.u32Limit, pCtx->gs.Attr.u));
469	}
470
471	PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
472	if (fFlags & HMSVM_LOG_LBR)
473	{
474	Log4(("%s: br_from=%#RX64 br_to=%#RX64 lastxcpt_from=%#RX64 lastxcpt_to=%#RX64\n", pszPrefix, pVmcbGuest->u64BR_FROM,
475	pVmcbGuest->u64BR_TO, pVmcbGuest->u64LASTEXCPFROM, pVmcbGuest->u64LASTEXCPTO));
476	}
477	NOREF(pszPrefix); NOREF(pVmcbGuest); NOREF(pCtx);
478	}
479	#endif /* VBOX_STRICT */
480
481
482	/**
483	* Sets up and activates AMD-V on the current CPU.
484	*
485	* @returns VBox status code.
486	* @param pHostCpu The HM physical-CPU structure.
487	* @param pVM The cross context VM structure. Can be
488	* NULL after a resume!
489	* @param pvCpuPage Pointer to the global CPU page.
490	* @param HCPhysCpuPage Physical address of the global CPU page.
491	* @param fEnabledByHost Whether the host OS has already initialized AMD-V.
492	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs (currently
493	* unused).
494	*/
495	VMMR0DECL(int) SVMR0EnableCpu(PHMPHYSCPU pHostCpu, PVMCC pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
496	PCSUPHWVIRTMSRS pHwvirtMsrs)
497	{
498	Assert(!fEnabledByHost);
499	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
500	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
501	Assert(pvCpuPage); NOREF(pvCpuPage);
502	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
503
504	RT_NOREF2(fEnabledByHost, pHwvirtMsrs);
505
506	/* Paranoid: Disable interrupt as, in theory, interrupt handlers might mess with EFER. */
507	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
508
509	/*
510	* We must turn on AMD-V and setup the host state physical address, as those MSRs are per CPU.
511	*/
512	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
513	if (u64HostEfer & MSR_K6_EFER_SVME)
514	{
515	/* If the VBOX_HWVIRTEX_IGNORE_SVM_IN_USE is active, then we blindly use AMD-V. */
516	if ( pVM
517	&& pVM->hm.s.svm.fIgnoreInUseError)
518	pHostCpu->fIgnoreAMDVInUseError = true;
519
520	if (!pHostCpu->fIgnoreAMDVInUseError)
521	{
522	ASMSetFlags(fEFlags);
523	return VERR_SVM_IN_USE;
524	}
525	}
526
527	/* Turn on AMD-V in the EFER MSR. */
528	ASMWrMsr(MSR_K6_EFER, u64HostEfer \| MSR_K6_EFER_SVME);
529
530	/* Write the physical page address where the CPU will store the host state while executing the VM. */
531	ASMWrMsr(MSR_K8_VM_HSAVE_PA, HCPhysCpuPage);
532
533	/* Restore interrupts. */
534	ASMSetFlags(fEFlags);
535
536	/*
537	* Theoretically, other hypervisors may have used ASIDs, ideally we should flush all
538	* non-zero ASIDs when enabling SVM. AMD doesn't have an SVM instruction to flush all
539	* ASIDs (flushing is done upon VMRUN). Therefore, flag that we need to flush the TLB
540	* entirely with before executing any guest code.
541	*/
542	pHostCpu->fFlushAsidBeforeUse = true;
543
544	/*
545	* Ensure each VCPU scheduled on this CPU gets a new ASID on resume. See @bugref{6255}.
546	*/
547	++pHostCpu->cTlbFlushes;
548
549	return VINF_SUCCESS;
550	}
551
552
553	/**
554	* Deactivates AMD-V on the current CPU.
555	*
556	* @returns VBox status code.
557	* @param pHostCpu The HM physical-CPU structure.
558	* @param pvCpuPage Pointer to the global CPU page.
559	* @param HCPhysCpuPage Physical address of the global CPU page.
560	*/
561	VMMR0DECL(int) SVMR0DisableCpu(PHMPHYSCPU pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
562	{
563	RT_NOREF1(pHostCpu);
564	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
565	AssertReturn( HCPhysCpuPage
566	&& HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
567	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
568
569	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with EFER. */
570	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
571
572	/* Turn off AMD-V in the EFER MSR. */
573	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
574	ASMWrMsr(MSR_K6_EFER, u64HostEfer & ~MSR_K6_EFER_SVME);
575
576	/* Invalidate host state physical address. */
577	ASMWrMsr(MSR_K8_VM_HSAVE_PA, 0);
578
579	/* Restore interrupts. */
580	ASMSetFlags(fEFlags);
581
582	return VINF_SUCCESS;
583	}
584
585
586	/**
587	* Does global AMD-V initialization (called during module initialization).
588	*
589	* @returns VBox status code.
590	*/
591	VMMR0DECL(int) SVMR0GlobalInit(void)
592	{
593	/*
594	* Allocate 12 KB (3 pages) for the IO bitmap. Since this is non-optional and we always
595	* intercept all IO accesses, it's done once globally here instead of per-VM.
596	*/
597	Assert(g_hMemObjIOBitmap == NIL_RTR0MEMOBJ);
598	int rc = RTR0MemObjAllocCont(&g_hMemObjIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, false /* fExecutable */);
599	if (RT_FAILURE(rc))
600	return rc;
601
602	g_pvIOBitmap = RTR0MemObjAddress(g_hMemObjIOBitmap);
603	g_HCPhysIOBitmap = RTR0MemObjGetPagePhysAddr(g_hMemObjIOBitmap, 0 /* iPage */);
604
605	/* Set all bits to intercept all IO accesses. */
606	ASMMemFill32(g_pvIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
607
608	return VINF_SUCCESS;
609	}
610
611
612	/**
613	* Does global AMD-V termination (called during module termination).
614	*/
615	VMMR0DECL(void) SVMR0GlobalTerm(void)
616	{
617	if (g_hMemObjIOBitmap != NIL_RTR0MEMOBJ)
618	{
619	RTR0MemObjFree(g_hMemObjIOBitmap, true /* fFreeMappings */);
620	g_pvIOBitmap = NULL;
621	g_HCPhysIOBitmap = 0;
622	g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
623	}
624	}
625
626
627	/**
628	* Frees any allocated per-VCPU structures for a VM.
629	*
630	* @param pVM The cross context VM structure.
631	*/
632	DECLINLINE(void) hmR0SvmFreeStructs(PVMCC pVM)
633	{
634	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
635	{
636	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
637	AssertPtr(pVCpu);
638
639	if (pVCpu->hmr0.s.svm.hMemObjVmcbHost != NIL_RTR0MEMOBJ)
640	{
641	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjVmcbHost, false);
642	pVCpu->hmr0.s.svm.HCPhysVmcbHost = 0;
643	pVCpu->hmr0.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
644	}
645
646	if (pVCpu->hmr0.s.svm.hMemObjVmcb != NIL_RTR0MEMOBJ)
647	{
648	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjVmcb, false);
649	pVCpu->hmr0.s.svm.pVmcb = NULL;
650	pVCpu->hmr0.s.svm.HCPhysVmcb = 0;
651	pVCpu->hmr0.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
652	}
653
654	if (pVCpu->hmr0.s.svm.hMemObjMsrBitmap != NIL_RTR0MEMOBJ)
655	{
656	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjMsrBitmap, false);
657	pVCpu->hmr0.s.svm.pvMsrBitmap = NULL;
658	pVCpu->hmr0.s.svm.HCPhysMsrBitmap = 0;
659	pVCpu->hmr0.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
660	}
661	}
662	}
663
664
665	/**
666	* Sets pfnVMRun to the best suited variant.
667	*
668	* This must be called whenever anything changes relative to the SVMR0VMRun
669	* variant selection:
670	* - pVCpu->hm.s.fLoadSaveGuestXcr0
671	* - CPUMCTX_WSF_IBPB_ENTRY in pVCpu->cpum.GstCtx.fWorldSwitcher
672	* - CPUMCTX_WSF_IBPB_EXIT in pVCpu->cpum.GstCtx.fWorldSwitcher
673	* - Perhaps: CPUMIsGuestFPUStateActive() (windows only)
674	* - Perhaps: CPUMCTX.fXStateMask (windows only)
675	*
676	* We currently ASSUME that neither CPUMCTX_WSF_IBPB_ENTRY nor
677	* CPUMCTX_WSF_IBPB_EXIT cannot be changed at runtime.
678	*/
679	static void hmR0SvmUpdateVmRunFunction(PVMCPUCC pVCpu)
680	{
681	static const struct CLANGWORKAROUND { PFNHMSVMVMRUN pfn; } s_aHmR0SvmVmRunFunctions[] =
682	{
683	{ hmR0SvmVmRun_SansXcr0_SansIbpbEntry_SansIbpbExit },
684	{ hmR0SvmVmRun_WithXcr0_SansIbpbEntry_SansIbpbExit },
685	{ hmR0SvmVmRun_SansXcr0_WithIbpbEntry_SansIbpbExit },
686	{ hmR0SvmVmRun_WithXcr0_WithIbpbEntry_SansIbpbExit },
687	{ hmR0SvmVmRun_SansXcr0_SansIbpbEntry_WithIbpbExit },
688	{ hmR0SvmVmRun_WithXcr0_SansIbpbEntry_WithIbpbExit },
689	{ hmR0SvmVmRun_SansXcr0_WithIbpbEntry_WithIbpbExit },
690	{ hmR0SvmVmRun_WithXcr0_WithIbpbEntry_WithIbpbExit },
691	};
692	uintptr_t const idx = (pVCpu->hmr0.s.fLoadSaveGuestXcr0 ? 1 : 0)
693	\| (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_IBPB_ENTRY ? 2 : 0)
694	\| (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_IBPB_EXIT ? 4 : 0);
695	PFNHMSVMVMRUN const pfnVMRun = s_aHmR0SvmVmRunFunctions[idx].pfn;
696	if (pVCpu->hmr0.s.svm.pfnVMRun != pfnVMRun)
697	pVCpu->hmr0.s.svm.pfnVMRun = pfnVMRun;
698	}
699
700
701	/**
702	* Selector FNHMSVMVMRUN implementation.
703	*/
704	static DECLCALLBACK(int) hmR0SvmVMRunSelector(PVMCC pVM, PVMCPUCC pVCpu, RTHCPHYS HCPhysVMCB)
705	{
706	hmR0SvmUpdateVmRunFunction(pVCpu);
707	return pVCpu->hmr0.s.svm.pfnVMRun(pVM, pVCpu, HCPhysVMCB);
708	}
709
710
711	/**
712	* Does per-VM AMD-V initialization.
713	*
714	* @returns VBox status code.
715	* @param pVM The cross context VM structure.
716	*/
717	VMMR0DECL(int) SVMR0InitVM(PVMCC pVM)
718	{
719	int rc = VERR_INTERNAL_ERROR_5;
720
721	/*
722	* Check for an AMD CPU erratum which requires us to flush the TLB before every world-switch.
723	*/
724	uint32_t u32Family;
725	uint32_t u32Model;
726	uint32_t u32Stepping;
727	if (HMIsSubjectToSvmErratum170(&u32Family, &u32Model, &u32Stepping))
728	{
729	Log4Func(("AMD cpu with erratum 170 family %#x model %#x stepping %#x\n", u32Family, u32Model, u32Stepping));
730	pVM->hmr0.s.svm.fAlwaysFlushTLB = true;
731	}
732
733	/*
734	* Initialize the R0 memory objects up-front so we can properly cleanup on allocation failures.
735	*/
736	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
737	{
738	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
739	pVCpu->hmr0.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
740	pVCpu->hmr0.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
741	pVCpu->hmr0.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
742	}
743
744	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
745	{
746	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
747
748	/*
749	* Initialize the hardware-assisted SVM guest-execution handler.
750	* We now use a single handler for both 32-bit and 64-bit guests, see @bugref{6208#c73}.
751	*/
752	pVCpu->hmr0.s.svm.pfnVMRun = hmR0SvmVMRunSelector;
753
754	/*
755	* Allocate one page for the host-context VM control block (VMCB). This is used for additional host-state (such as
756	* FS, GS, Kernel GS Base, etc.) apart from the host-state save area specified in MSR_K8_VM_HSAVE_PA.
757	*/
758	/** @todo Does this need to be below 4G? */
759	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjVmcbHost, SVM_VMCB_PAGES << HOST_PAGE_SHIFT, false /* fExecutable */);
760	if (RT_FAILURE(rc))
761	goto failure_cleanup;
762
763	void *pvVmcbHost = RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjVmcbHost);
764	pVCpu->hmr0.s.svm.HCPhysVmcbHost = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjVmcbHost, 0 /* iPage */);
765	Assert(pVCpu->hmr0.s.svm.HCPhysVmcbHost < _4G);
766	RT_BZERO(pvVmcbHost, HOST_PAGE_SIZE);
767
768	/*
769	* Allocate one page for the guest-state VMCB.
770	*/
771	/** @todo Does this need to be below 4G? */
772	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjVmcb, SVM_VMCB_PAGES << HOST_PAGE_SHIFT, false /* fExecutable */);
773	if (RT_FAILURE(rc))
774	goto failure_cleanup;
775
776	pVCpu->hmr0.s.svm.pVmcb = (PSVMVMCB)RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjVmcb);
777	pVCpu->hmr0.s.svm.HCPhysVmcb = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjVmcb, 0 /* iPage */);
778	Assert(pVCpu->hmr0.s.svm.HCPhysVmcb < _4G);
779	RT_BZERO(pVCpu->hmr0.s.svm.pVmcb, HOST_PAGE_SIZE);
780
781	/*
782	* Allocate two pages (8 KB) for the MSR permission bitmap. There doesn't seem to be a way to convince
783	* SVM to not require one.
784	*/
785	/** @todo Does this need to be below 4G? */
786	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjMsrBitmap, SVM_MSRPM_PAGES << HOST_PAGE_SHIFT,
787	false /* fExecutable */);
788	if (RT_FAILURE(rc))
789	goto failure_cleanup;
790
791	pVCpu->hmr0.s.svm.pvMsrBitmap = RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjMsrBitmap);
792	pVCpu->hmr0.s.svm.HCPhysMsrBitmap = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjMsrBitmap, 0 /* iPage */);
793	/* Set all bits to intercept all MSR accesses (changed later on). */
794	ASMMemFill32(pVCpu->hmr0.s.svm.pvMsrBitmap, SVM_MSRPM_PAGES << HOST_PAGE_SHIFT, UINT32_C(0xffffffff));
795	}
796
797	return VINF_SUCCESS;
798
799	failure_cleanup:
800	hmR0SvmFreeStructs(pVM);
801	return rc;
802	}
803
804
805	/**
806	* Does per-VM AMD-V termination.
807	*
808	* @returns VBox status code.
809	* @param pVM The cross context VM structure.
810	*/
811	VMMR0DECL(int) SVMR0TermVM(PVMCC pVM)
812	{
813	hmR0SvmFreeStructs(pVM);
814	return VINF_SUCCESS;
815	}
816
817
818	/**
819	* Returns whether the VMCB Clean Bits feature is supported.
820	*
821	* @returns @c true if supported, @c false otherwise.
822	* @param pVCpu The cross context virtual CPU structure.
823	* @param fIsNestedGuest Whether we are currently executing the nested-guest.
824	*/
825	DECL_FORCE_INLINE(bool) hmR0SvmSupportsVmcbCleanBits(PVMCPUCC pVCpu, bool fIsNestedGuest)
826	{
827	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
828	bool const fHostVmcbCleanBits = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VMCB_CLEAN);
829	if (!fIsNestedGuest)
830	return fHostVmcbCleanBits;
831	return fHostVmcbCleanBits && pVM->cpum.ro.GuestFeatures.fSvmVmcbClean;
832	}
833
834
835	/**
836	* Returns whether the decode assists feature is supported.
837	*
838	* @returns @c true if supported, @c false otherwise.
839	* @param pVCpu The cross context virtual CPU structure.
840	*/
841	DECLINLINE(bool) hmR0SvmSupportsDecodeAssists(PVMCPUCC pVCpu)
842	{
843	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
844	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
845	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
846	return (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS)
847	&& pVM->cpum.ro.GuestFeatures.fSvmDecodeAssists;
848	#endif
849	return RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS);
850	}
851
852
853	/**
854	* Returns whether the NRIP_SAVE feature is supported.
855	*
856	* @returns @c true if supported, @c false otherwise.
857	* @param pVCpu The cross context virtual CPU structure.
858	*/
859	DECLINLINE(bool) hmR0SvmSupportsNextRipSave(PVMCPUCC pVCpu)
860	{
861	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
862	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
863	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
864	return (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE)
865	&& pVM->cpum.ro.GuestFeatures.fSvmNextRipSave;
866	#endif
867	return RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE);
868	}
869
870
871	/**
872	* Sets the permission bits for the specified MSR in the MSRPM bitmap.
873	*
874	* @param pVCpu The cross context virtual CPU structure.
875	* @param pbMsrBitmap Pointer to the MSR bitmap.
876	* @param idMsr The MSR for which the permissions are being set.
877	* @param enmRead MSR read permissions.
878	* @param enmWrite MSR write permissions.
879	*
880	* @remarks This function does -not- clear the VMCB clean bits for MSRPM. The
881	* caller needs to take care of this.
882	*/
883	static void hmR0SvmSetMsrPermission(PVMCPUCC pVCpu, uint8_t *pbMsrBitmap, uint32_t idMsr, SVMMSREXITREAD enmRead,
884	SVMMSREXITWRITE enmWrite)
885	{
886	bool const fInNestedGuestMode = CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx);
887	uint16_t offMsrpm;
888	uint8_t uMsrpmBit;
889	int rc = CPUMGetSvmMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
890	AssertRC(rc);
891
892	Assert(uMsrpmBit == 0 \|\| uMsrpmBit == 2 \|\| uMsrpmBit == 4 \|\| uMsrpmBit == 6);
893	Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
894
895	pbMsrBitmap += offMsrpm;
896	if (enmRead == SVMMSREXIT_INTERCEPT_READ)
897	*pbMsrBitmap \|= RT_BIT(uMsrpmBit);
898	else
899	{
900	if (!fInNestedGuestMode)
901	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
902	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
903	else
904	{
905	/* Only clear the bit if the nested-guest is also not intercepting the MSR read.*/
906	if (!(pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm] & RT_BIT(uMsrpmBit)))
907	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
908	else
909	Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit));
910	}
911	#endif
912	}
913
914	if (enmWrite == SVMMSREXIT_INTERCEPT_WRITE)
915	*pbMsrBitmap \|= RT_BIT(uMsrpmBit + 1);
916	else
917	{
918	if (!fInNestedGuestMode)
919	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
920	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
921	else
922	{
923	/* Only clear the bit if the nested-guest is also not intercepting the MSR write.*/
924	if (!(pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm] & RT_BIT(uMsrpmBit + 1)))
925	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
926	else
927	Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
928	}
929	#endif
930	}
931	}
932
933
934	/**
935	* Sets up AMD-V for the specified VM.
936	* This function is only called once per-VM during initalization.
937	*
938	* @returns VBox status code.
939	* @param pVM The cross context VM structure.
940	*/
941	VMMR0DECL(int) SVMR0SetupVM(PVMCC pVM)
942	{
943	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
944	AssertReturn(pVM, VERR_INVALID_PARAMETER);
945
946	/*
947	* Validate and copy over some parameters.
948	*/
949	AssertReturn(pVM->hm.s.svm.fSupported, VERR_INCOMPATIBLE_CONFIG);
950	bool const fNestedPaging = pVM->hm.s.fNestedPagingCfg;
951	AssertReturn(!fNestedPaging \|\| (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NESTED_PAGING), VERR_INCOMPATIBLE_CONFIG);
952	pVM->hmr0.s.fNestedPaging = fNestedPaging;
953	pVM->hmr0.s.fAllow64BitGuests = pVM->hm.s.fAllow64BitGuestsCfg;
954
955	/*
956	* Determin some configuration parameters.
957	*/
958	bool const fPauseFilter = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER);
959	bool const fPauseFilterThreshold = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER_THRESHOLD);
960	bool const fUsePauseFilter = fPauseFilter && pVM->hm.s.svm.cPauseFilter;
961
962	bool const fLbrVirt = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_LBR_VIRT);
963	bool const fUseLbrVirt = fLbrVirt && pVM->hm.s.svm.fLbrVirt; /** @todo IEM implementation etc. */
964
965	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
966	bool const fVirtVmsaveVmload = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VIRT_VMSAVE_VMLOAD);
967	bool const fUseVirtVmsaveVmload = fVirtVmsaveVmload && pVM->hm.s.svm.fVirtVmsaveVmload && fNestedPaging;
968
969	bool const fVGif = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VGIF);
970	bool const fUseVGif = fVGif && pVM->hm.s.svm.fVGif;
971	#endif
972
973	PVMCPUCC pVCpu0 = VMCC_GET_CPU_0(pVM);
974	PSVMVMCB pVmcb0 = pVCpu0->hmr0.s.svm.pVmcb;
975	AssertMsgReturn(RT_VALID_PTR(pVmcb0), ("Invalid pVmcb (%p) for vcpu[0]\n", pVmcb0), VERR_SVM_INVALID_PVMCB);
976	PSVMVMCBCTRL pVmcbCtrl0 = &pVmcb0->ctrl;
977
978	/* Always trap #AC for reasons of security. */
979	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_AC);
980
981	/* Always trap #DB for reasons of security. */
982	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_DB);
983
984	/* Trap exceptions unconditionally (debug purposes). */
985	#ifdef HMSVM_ALWAYS_TRAP_PF
986	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_PF);
987	#endif
988	#ifdef HMSVM_ALWAYS_TRAP_ALL_XCPTS
989	/* If you add any exceptions here, make sure to update hmR0SvmHandleExit(). */
990	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_BP)
991	\| RT_BIT_32(X86_XCPT_DE)
992	\| RT_BIT_32(X86_XCPT_NM)
993	\| RT_BIT_32(X86_XCPT_UD)
994	\| RT_BIT_32(X86_XCPT_NP)
995	\| RT_BIT_32(X86_XCPT_SS)
996	\| RT_BIT_32(X86_XCPT_GP)
997	\| RT_BIT_32(X86_XCPT_PF)
998	\| RT_BIT_32(X86_XCPT_MF)
999	;
1000	#endif
1001
1002	/* Apply the exceptions intercepts needed by the GIM provider. */
1003	if (pVCpu0->hm.s.fGIMTrapXcptUD \|\| pVCpu0->hm.s.svm.fEmulateLongModeSysEnterExit)
1004	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_UD);
1005
1006	/* Apply the exceptions intercepts needed by the GCM fixers. */
1007	if (pVCpu0->hm.s.fGCMTrapXcptDE)
1008	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_DE);
1009
1010	/* The mesa 3d driver hack needs #GP. */
1011	if (pVCpu0->hm.s.fTrapXcptGpForLovelyMesaDrv)
1012	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_GP);
1013
1014	/* Set up unconditional intercepts and conditions. */
1015	pVmcbCtrl0->u64InterceptCtrl = HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS
1016	\| SVM_CTRL_INTERCEPT_VMMCALL
1017	\| SVM_CTRL_INTERCEPT_VMSAVE
1018	\| SVM_CTRL_INTERCEPT_VMLOAD
1019	\| SVM_CTRL_INTERCEPT_CLGI
1020	\| SVM_CTRL_INTERCEPT_STGI;
1021
1022	#ifdef HMSVM_ALWAYS_TRAP_TASK_SWITCH
1023	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_TASK_SWITCH;
1024	#endif
1025
1026	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1027	if (pVCpu0->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvm)
1028	{
1029	/* Virtualized VMSAVE/VMLOAD. */
1030	if (fUseVirtVmsaveVmload)
1031	{
1032	pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload = 1;
1033	pVmcbCtrl0->u64InterceptCtrl &= ~( SVM_CTRL_INTERCEPT_VMSAVE
1034	\| SVM_CTRL_INTERCEPT_VMLOAD);
1035	}
1036	else
1037	Assert(!pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload);
1038
1039	/* Virtual GIF. */
1040	if (fUseVGif)
1041	{
1042	pVmcbCtrl0->IntCtrl.n.u1VGifEnable = 1;
1043	pVmcbCtrl0->u64InterceptCtrl &= ~( SVM_CTRL_INTERCEPT_CLGI
1044	\| SVM_CTRL_INTERCEPT_STGI);
1045	}
1046	else
1047	Assert(!pVmcbCtrl0->IntCtrl.n.u1VGifEnable);
1048	}
1049	else
1050	#endif
1051	{
1052	Assert(!pVCpu0->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvm);
1053	Assert(!pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload);
1054	Assert(!pVmcbCtrl0->IntCtrl.n.u1VGifEnable);
1055	}
1056
1057	/* CR4 writes must always be intercepted for tracking PGM mode changes and
1058	AVX (for XCR0 syncing during worlds switching). */
1059	pVmcbCtrl0->u16InterceptWrCRx = RT_BIT(4);
1060
1061	/* Intercept all DRx reads and writes by default. Changed later on. */
1062	pVmcbCtrl0->u16InterceptRdDRx = 0xffff;
1063	pVmcbCtrl0->u16InterceptWrDRx = 0xffff;
1064
1065	/* Virtualize masking of INTR interrupts. (reads/writes from/to CR8 go to the V_TPR register) */
1066	pVmcbCtrl0->IntCtrl.n.u1VIntrMasking = 1;
1067
1068	/* Ignore the priority in the virtual TPR. This is necessary for delivering PIC style (ExtInt) interrupts
1069	and we currently deliver both PIC and APIC interrupts alike, see hmR0SvmEvaluatePendingEvent() */
1070	pVmcbCtrl0->IntCtrl.n.u1IgnoreTPR = 1;
1071
1072	/* Set the IO permission bitmap physical addresses. */
1073	pVmcbCtrl0->u64IOPMPhysAddr = g_HCPhysIOBitmap;
1074
1075	/* LBR virtualization. */
1076	pVmcbCtrl0->LbrVirt.n.u1LbrVirt = fUseLbrVirt;
1077
1078	/* The host ASID MBZ, for the guest start with 1. */
1079	pVmcbCtrl0->TLBCtrl.n.u32ASID = 1;
1080
1081	/* Setup Nested Paging. This doesn't change throughout the execution time of the VM. */
1082	pVmcbCtrl0->NestedPagingCtrl.n.u1NestedPaging = fNestedPaging;
1083
1084	/* Without Nested Paging, we need additionally intercepts. */
1085	if (!fNestedPaging)
1086	{
1087	/* CR3 reads/writes must be intercepted; our shadow values differ from the guest values. */
1088	pVmcbCtrl0->u16InterceptRdCRx \|= RT_BIT(3);
1089	pVmcbCtrl0->u16InterceptWrCRx \|= RT_BIT(3);
1090
1091	/* Intercept INVLPG and task switches (may change CR3, EFLAGS, LDT). */
1092	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_INVLPG
1093	\| SVM_CTRL_INTERCEPT_TASK_SWITCH;
1094
1095	/* Page faults must be intercepted to implement shadow paging. */
1096	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_PF);
1097	}
1098
1099	/* Workaround for missing OS/2 TLB flush, see ticketref:20625. */
1100	if (pVM->hm.s.fMissingOS2TlbFlushWorkaround)
1101	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_TR_WRITES;
1102
1103	/* Setup Pause Filter for guest pause-loop (spinlock) exiting. */
1104	if (fUsePauseFilter)
1105	{
1106	Assert(pVM->hm.s.svm.cPauseFilter > 0);
1107	pVmcbCtrl0->u16PauseFilterCount = pVM->hm.s.svm.cPauseFilter;
1108	if (fPauseFilterThreshold)
1109	pVmcbCtrl0->u16PauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
1110	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_PAUSE;
1111	}
1112
1113	/*
1114	* Setup the MSR permission bitmap.
1115	* The following MSRs are saved/restored automatically during the world-switch.
1116	* Don't intercept guest read/write accesses to these MSRs.
1117	*/
1118	uint8_t pbMsrBitmap0 = (uint8_t )pVCpu0->hmr0.s.svm.pvMsrBitmap;
1119	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1120	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_CSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1121	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K6_STAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1122	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_SF_MASK, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1123	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_FS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1124	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1125	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_KERNEL_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1126	if (!pVCpu0->hm.s.svm.fEmulateLongModeSysEnterExit)
1127	{
1128	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_CS, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1129	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1130	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1131	}
1132	else
1133	{
1134	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_CS, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1135	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1136	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1137	}
1138	pVmcbCtrl0->u64MSRPMPhysAddr = pVCpu0->hmr0.s.svm.HCPhysMsrBitmap;
1139
1140	/* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
1141	Assert(pVmcbCtrl0->u32VmcbCleanBits == 0);
1142
1143	for (VMCPUID idCpu = 1; idCpu < pVM->cCpus; idCpu++)
1144	{
1145	PVMCPUCC pVCpuCur = VMCC_GET_CPU(pVM, idCpu);
1146	PSVMVMCB pVmcbCur = pVCpuCur->hmr0.s.svm.pVmcb;
1147	AssertMsgReturn(RT_VALID_PTR(pVmcbCur), ("Invalid pVmcb (%p) for vcpu[%u]\n", pVmcbCur, idCpu), VERR_SVM_INVALID_PVMCB);
1148	PSVMVMCBCTRL pVmcbCtrlCur = &pVmcbCur->ctrl;
1149
1150	/* Copy the VMCB control area. */
1151	memcpy(pVmcbCtrlCur, pVmcbCtrl0, sizeof(*pVmcbCtrlCur));
1152
1153	/* Copy the MSR bitmap and setup the VCPU-specific host physical address. */
1154	uint8_t pbMsrBitmapCur = (uint8_t )pVCpuCur->hmr0.s.svm.pvMsrBitmap;
1155	memcpy(pbMsrBitmapCur, pbMsrBitmap0, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
1156	pVmcbCtrlCur->u64MSRPMPhysAddr = pVCpuCur->hmr0.s.svm.HCPhysMsrBitmap;
1157
1158	/* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
1159	Assert(pVmcbCtrlCur->u32VmcbCleanBits == 0);
1160
1161	/* Verify our assumption that GIM providers trap #UD uniformly across VCPUs initially. */
1162	Assert(pVCpuCur->hm.s.fGIMTrapXcptUD == pVCpu0->hm.s.fGIMTrapXcptUD);
1163	/* Same for GCM, #DE trapping should be uniform across VCPUs. */
1164	Assert(pVCpuCur->hm.s.fGCMTrapXcptDE == pVCpu0->hm.s.fGCMTrapXcptDE);
1165	}
1166
1167	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1168	LogRel(("HM: fUsePauseFilter=%RTbool fUseLbrVirt=%RTbool fUseVGif=%RTbool fUseVirtVmsaveVmload=%RTbool\n", fUsePauseFilter,
1169	fUseLbrVirt, fUseVGif, fUseVirtVmsaveVmload));
1170	#else
1171	LogRel(("HM: fUsePauseFilter=%RTbool fUseLbrVirt=%RTbool\n", fUsePauseFilter, fUseLbrVirt));
1172	#endif
1173	return VINF_SUCCESS;
1174	}
1175
1176
1177	/**
1178	* Gets a pointer to the currently active guest (or nested-guest) VMCB.
1179	*
1180	* @returns Pointer to the current context VMCB.
1181	* @param pVCpu The cross context virtual CPU structure.
1182	*/
1183	DECLINLINE(PSVMVMCB) hmR0SvmGetCurrentVmcb(PVMCPUCC pVCpu)
1184	{
1185	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1186	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
1187	return &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
1188	#endif
1189	return pVCpu->hmr0.s.svm.pVmcb;
1190	}
1191
1192
1193	/**
1194	* Gets a pointer to the nested-guest VMCB cache.
1195	*
1196	* @returns Pointer to the nested-guest VMCB cache.
1197	* @param pVCpu The cross context virtual CPU structure.
1198	*/
1199	DECLINLINE(PSVMNESTEDVMCBCACHE) hmR0SvmGetNestedVmcbCache(PVMCPUCC pVCpu)
1200	{
1201	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1202	Assert(pVCpu->hm.s.svm.NstGstVmcbCache.fCacheValid);
1203	return &pVCpu->hm.s.svm.NstGstVmcbCache;
1204	#else
1205	RT_NOREF(pVCpu);
1206	return NULL;
1207	#endif
1208	}
1209
1210
1211	/**
1212	* Invalidates a guest page by guest virtual address.
1213	*
1214	* @returns VBox status code.
1215	* @param pVCpu The cross context virtual CPU structure.
1216	* @param GCVirt Guest virtual address of the page to invalidate.
1217	*/
1218	VMMR0DECL(int) SVMR0InvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCVirt)
1219	{
1220	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fSupported);
1221
1222	bool const fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH) \|\| pVCpu->CTX_SUFF(pVM)->hmr0.s.svm.fAlwaysFlushTLB;
1223
1224	/* Skip it if a TLB flush is already pending. */
1225	if (!fFlushPending)
1226	{
1227	Log4Func(("%#RGv\n", GCVirt));
1228
1229	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
1230	AssertMsgReturn(pVmcb, ("Invalid pVmcb!\n"), VERR_SVM_INVALID_PVMCB);
1231
1232	SVMR0InvlpgA(GCVirt, pVmcb->ctrl.TLBCtrl.n.u32ASID);
1233	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1234	}
1235	return VINF_SUCCESS;
1236	}
1237
1238
1239	/**
1240	* Flushes the appropriate tagged-TLB entries.
1241	*
1242	* @param pHostCpu The HM physical-CPU structure.
1243	* @param pVCpu The cross context virtual CPU structure.
1244	* @param pVmcb Pointer to the VM control block.
1245	*/
1246	static void hmR0SvmFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1247	{
1248	/*
1249	* Force a TLB flush for the first world switch if the current CPU differs from the one
1250	* we ran on last. This can happen both for start & resume due to long jumps back to
1251	* ring-3.
1252	*
1253	* We also force a TLB flush every time when executing a nested-guest VCPU as there is no
1254	* correlation between it and the physical CPU.
1255	*
1256	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while
1257	* flushing the TLB, so we cannot reuse the ASIDs without flushing.
1258	*/
1259	bool fNewAsid = false;
1260	Assert(pHostCpu->idCpu != NIL_RTCPUID);
1261	if ( pVCpu->hmr0.s.idLastCpu != pHostCpu->idCpu
1262	\|\| pVCpu->hmr0.s.cTlbFlushes != pHostCpu->cTlbFlushes
1263	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1264	\|\| CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx)
1265	#endif
1266	)
1267	{
1268	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1269	pVCpu->hmr0.s.fForceTLBFlush = true;
1270	fNewAsid = true;
1271	}
1272
1273	/* Set TLB flush state as checked until we return from the world switch. */
1274	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true);
1275
1276	/* Check for explicit TLB flushes. */
1277	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1278	{
1279	pVCpu->hmr0.s.fForceTLBFlush = true;
1280	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1281	}
1282
1283	/*
1284	* If the AMD CPU erratum 170, We need to flush the entire TLB for each world switch. Sad.
1285	* This Host CPU requirement takes precedence.
1286	*/
1287	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1288	if (pVM->hmr0.s.svm.fAlwaysFlushTLB)
1289	{
1290	pHostCpu->uCurrentAsid = 1;
1291	pVCpu->hmr0.s.uCurrentAsid = 1;
1292	pVCpu->hmr0.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1293	pVCpu->hmr0.s.idLastCpu = pHostCpu->idCpu;
1294	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1295
1296	/* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1297	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1298	}
1299	else
1300	{
1301	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_NOTHING;
1302	if (pVCpu->hmr0.s.fForceTLBFlush)
1303	{
1304	/* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1305	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1306
1307	if (fNewAsid)
1308	{
1309	++pHostCpu->uCurrentAsid;
1310
1311	bool fHitASIDLimit = false;
1312	if (pHostCpu->uCurrentAsid >= g_uHmMaxAsid)
1313	{
1314	pHostCpu->uCurrentAsid = 1; /* Wraparound at 1; host uses 0 */
1315	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new ASID. */
1316	fHitASIDLimit = true;
1317	}
1318
1319	if ( fHitASIDLimit
1320	\|\| pHostCpu->fFlushAsidBeforeUse)
1321	{
1322	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1323	pHostCpu->fFlushAsidBeforeUse = false;
1324	}
1325
1326	pVCpu->hmr0.s.uCurrentAsid = pHostCpu->uCurrentAsid;
1327	pVCpu->hmr0.s.idLastCpu = pHostCpu->idCpu;
1328	pVCpu->hmr0.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1329	}
1330	else
1331	{
1332	if (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_FLUSH_BY_ASID)
1333	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_SINGLE_CONTEXT;
1334	else
1335	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1336	}
1337
1338	pVCpu->hmr0.s.fForceTLBFlush = false;
1339	}
1340	}
1341
1342	/* Update VMCB with the ASID. */
1343	if (pVmcb->ctrl.TLBCtrl.n.u32ASID != pVCpu->hmr0.s.uCurrentAsid)
1344	{
1345	pVmcb->ctrl.TLBCtrl.n.u32ASID = pVCpu->hmr0.s.uCurrentAsid;
1346	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_ASID;
1347	}
1348
1349	AssertMsg(pVCpu->hmr0.s.idLastCpu == pHostCpu->idCpu,
1350	("vcpu idLastCpu=%u hostcpu idCpu=%u\n", pVCpu->hmr0.s.idLastCpu, pHostCpu->idCpu));
1351	AssertMsg(pVCpu->hmr0.s.cTlbFlushes == pHostCpu->cTlbFlushes,
1352	("Flush count mismatch for cpu %u (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hmr0.s.cTlbFlushes, pHostCpu->cTlbFlushes));
1353	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < g_uHmMaxAsid,
1354	("cpu%d uCurrentAsid = %x\n", pHostCpu->idCpu, pHostCpu->uCurrentAsid));
1355	AssertMsg(pVCpu->hmr0.s.uCurrentAsid >= 1 && pVCpu->hmr0.s.uCurrentAsid < g_uHmMaxAsid,
1356	("cpu%d VM uCurrentAsid = %x\n", pHostCpu->idCpu, pVCpu->hmr0.s.uCurrentAsid));
1357
1358	#ifdef VBOX_WITH_STATISTICS
1359	if (pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_NOTHING)
1360	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1361	else if ( pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT
1362	\|\| pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT_RETAIN_GLOBALS)
1363	{
1364	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1365	}
1366	else
1367	{
1368	Assert(pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_ENTIRE);
1369	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushEntire);
1370	}
1371	#endif
1372	}
1373
1374
1375	/**
1376	* Sets an exception intercept in the specified VMCB.
1377	*
1378	* @param pVmcb Pointer to the VM control block.
1379	* @param uXcpt The exception (X86_XCPT_*).
1380	*/
1381	DECLINLINE(void) hmR0SvmSetXcptIntercept(PSVMVMCB pVmcb, uint8_t uXcpt)
1382	{
1383	if (!(pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt)))
1384	{
1385	pVmcb->ctrl.u32InterceptXcpt \|= RT_BIT(uXcpt);
1386	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1387	}
1388	}
1389
1390
1391	/**
1392	* Clears an exception intercept in the specified VMCB.
1393	*
1394	* @param pVCpu The cross context virtual CPU structure.
1395	* @param pVmcb Pointer to the VM control block.
1396	* @param uXcpt The exception (X86_XCPT_*).
1397	*
1398	* @remarks This takes into account if we're executing a nested-guest and only
1399	* removes the exception intercept if both the guest -and- nested-guest
1400	* are not intercepting it.
1401	*/
1402	DECLINLINE(void) hmR0SvmClearXcptIntercept(PVMCPUCC pVCpu, PSVMVMCB pVmcb, uint8_t uXcpt)
1403	{
1404	Assert(uXcpt != X86_XCPT_DB);
1405	Assert(uXcpt != X86_XCPT_AC);
1406	Assert(uXcpt != X86_XCPT_GP);
1407	#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
1408	if (pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt))
1409	{
1410	bool fRemove = true;
1411	# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1412	/* Only remove the intercept if the nested-guest is also not intercepting it! */
1413	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1414	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1415	{
1416	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1417	fRemove = !(pVmcbNstGstCache->u32InterceptXcpt & RT_BIT(uXcpt));
1418	}
1419	# else
1420	RT_NOREF(pVCpu);
1421	# endif
1422	if (fRemove)
1423	{
1424	pVmcb->ctrl.u32InterceptXcpt &= ~RT_BIT(uXcpt);
1425	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1426	}
1427	}
1428	#else
1429	RT_NOREF3(pVCpu, pVmcb, uXcpt);
1430	#endif
1431	}
1432
1433
1434	/**
1435	* Sets a control intercept in the specified VMCB.
1436	*
1437	* @param pVmcb Pointer to the VM control block.
1438	* @param fCtrlIntercept The control intercept (SVM_CTRL_INTERCEPT_*).
1439	*/
1440	DECLINLINE(void) hmR0SvmSetCtrlIntercept(PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
1441	{
1442	if (!(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept))
1443	{
1444	pVmcb->ctrl.u64InterceptCtrl \|= fCtrlIntercept;
1445	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1446	}
1447	}
1448
1449
1450	/**
1451	* Clears a control intercept in the specified VMCB.
1452	*
1453	* @returns @c true if the intercept is still set, @c false otherwise.
1454	* @param pVCpu The cross context virtual CPU structure.
1455	* @param pVmcb Pointer to the VM control block.
1456	* @param fCtrlIntercept The control intercept (SVM_CTRL_INTERCEPT_*).
1457	*
1458	* @remarks This takes into account if we're executing a nested-guest and only
1459	* removes the control intercept if both the guest -and- nested-guest
1460	* are not intercepting it.
1461	*/
1462	static bool hmR0SvmClearCtrlIntercept(PVMCPUCC pVCpu, PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
1463	{
1464	if (pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept)
1465	{
1466	bool fRemove = true;
1467	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1468	/* Only remove the control intercept if the nested-guest is also not intercepting it! */
1469	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
1470	{
1471	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1472	fRemove = !(pVmcbNstGstCache->u64InterceptCtrl & fCtrlIntercept);
1473	}
1474	#else
1475	RT_NOREF(pVCpu);
1476	#endif
1477	if (fRemove)
1478	{
1479	pVmcb->ctrl.u64InterceptCtrl &= ~fCtrlIntercept;
1480	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1481	}
1482	}
1483
1484	return RT_BOOL(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept);
1485	}
1486
1487
1488	/**
1489	* Exports the guest (or nested-guest) CR0 into the VMCB.
1490	*
1491	* @param pVCpu The cross context virtual CPU structure.
1492	* @param pVmcb Pointer to the VM control block.
1493	*
1494	* @remarks This assumes we always pre-load the guest FPU.
1495	* @remarks No-long-jump zone!!!
1496	*/
1497	static void hmR0SvmExportGuestCR0(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1498	{
1499	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1500
1501	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1502	uint64_t const uGuestCr0 = pCtx->cr0;
1503	uint64_t uShadowCr0 = uGuestCr0;
1504
1505	/* Always enable caching. */
1506	uShadowCr0 &= ~(X86_CR0_CD \| X86_CR0_NW);
1507
1508	/* When Nested Paging is not available use shadow page tables and intercept #PFs (latter done in SVMR0SetupVM()). */
1509	if (!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
1510	{
1511	uShadowCr0 \|= X86_CR0_PG /* Use shadow page tables. */
1512	\| X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF #VMEXIT. */
1513	}
1514
1515	/*
1516	* Use the #MF style of legacy-FPU error reporting for now. Although AMD-V has MSRs that
1517	* lets us isolate the host from it, IEM/REM still needs work to emulate it properly,
1518	* see @bugref{7243#c103}.
1519	*/
1520	if (!(uGuestCr0 & X86_CR0_NE))
1521	{
1522	uShadowCr0 \|= X86_CR0_NE;
1523	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_MF);
1524	}
1525	else
1526	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_MF);
1527
1528	/*
1529	* If the shadow and guest CR0 are identical we can avoid intercepting CR0 reads.
1530	*
1531	* CR0 writes still needs interception as PGM requires tracking paging mode changes,
1532	* see @bugref{6944}.
1533	*
1534	* We also don't ever want to honor weird things like cache disable from the guest.
1535	* However, we can avoid intercepting changes to the TS & MP bits by clearing the CR0
1536	* write intercept below and keeping SVM_CTRL_INTERCEPT_CR0_SEL_WRITE instead.
1537	*/
1538	if (uShadowCr0 == uGuestCr0)
1539	{
1540	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1541	{
1542	pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(0);
1543	pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(0);
1544	Assert(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_CR0_SEL_WRITE);
1545	}
1546	else
1547	{
1548	/* If the nested-hypervisor intercepts CR0 reads/writes, we need to continue intercepting them. */
1549	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1550	pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx & ~RT_BIT(0))
1551	\| (pVmcbNstGstCache->u16InterceptRdCRx & RT_BIT(0));
1552	pVmcb->ctrl.u16InterceptWrCRx = (pVmcb->ctrl.u16InterceptWrCRx & ~RT_BIT(0))
1553	\| (pVmcbNstGstCache->u16InterceptWrCRx & RT_BIT(0));
1554	}
1555	}
1556	else
1557	{
1558	pVmcb->ctrl.u16InterceptRdCRx \|= RT_BIT(0);
1559	pVmcb->ctrl.u16InterceptWrCRx \|= RT_BIT(0);
1560	}
1561	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1562
1563	Assert(!RT_HI_U32(uShadowCr0));
1564	if (pVmcb->guest.u64CR0 != uShadowCr0)
1565	{
1566	pVmcb->guest.u64CR0 = uShadowCr0;
1567	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1568	}
1569	}
1570
1571
1572	/**
1573	* Exports the guest (or nested-guest) CR3 into the VMCB.
1574	*
1575	* @param pVCpu The cross context virtual CPU structure.
1576	* @param pVmcb Pointer to the VM control block.
1577	*
1578	* @remarks No-long-jump zone!!!
1579	*/
1580	static void hmR0SvmExportGuestCR3(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1581	{
1582	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1583
1584	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1585	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1586	if (pVM->hmr0.s.fNestedPaging)
1587	{
1588	pVmcb->ctrl.u64NestedPagingCR3 = PGMGetHyperCR3(pVCpu);
1589	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1590	pVmcb->guest.u64CR3 = pCtx->cr3;
1591	Assert(pVmcb->ctrl.u64NestedPagingCR3);
1592	}
1593	else
1594	pVmcb->guest.u64CR3 = PGMGetHyperCR3(pVCpu);
1595
1596	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1597	}
1598
1599
1600	/**
1601	* Exports the guest (or nested-guest) CR4 into the VMCB.
1602	*
1603	* @param pVCpu The cross context virtual CPU structure.
1604	* @param pVmcb Pointer to the VM control block.
1605	*
1606	* @remarks No-long-jump zone!!!
1607	*/
1608	static int hmR0SvmExportGuestCR4(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1609	{
1610	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1611
1612	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1613	uint64_t uShadowCr4 = pCtx->cr4;
1614	if (!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
1615	{
1616	switch (pVCpu->hm.s.enmShadowMode)
1617	{
1618	case PGMMODE_REAL:
1619	case PGMMODE_PROTECTED: /* Protected mode, no paging. */
1620	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1621
1622	case PGMMODE_32_BIT: /* 32-bit paging. */
1623	uShadowCr4 &= ~X86_CR4_PAE;
1624	break;
1625
1626	case PGMMODE_PAE: /* PAE paging. */
1627	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1628	/** Must use PAE paging as we could use physical memory > 4 GB */
1629	uShadowCr4 \|= X86_CR4_PAE;
1630	break;
1631
1632	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1633	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1634	#ifdef VBOX_WITH_64_BITS_GUESTS
1635	break;
1636	#else
1637	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1638	#endif
1639
1640	default: /* shut up gcc */
1641	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1642	}
1643	}
1644
1645	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
1646	bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
1647	if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
1648	{
1649	pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
1650	hmR0SvmUpdateVmRunFunction(pVCpu);
1651	}
1652
1653	/* Avoid intercepting CR4 reads if the guest and shadow CR4 values are identical. */
1654	if (uShadowCr4 == pCtx->cr4)
1655	{
1656	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1657	pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(4);
1658	else
1659	{
1660	/* If the nested-hypervisor intercepts CR4 reads, we need to continue intercepting them. */
1661	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1662	pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx & ~RT_BIT(4))
1663	\| (pVmcbNstGstCache->u16InterceptRdCRx & RT_BIT(4));
1664	}
1665	}
1666	else
1667	pVmcb->ctrl.u16InterceptRdCRx \|= RT_BIT(4);
1668
1669	/* CR4 writes are always intercepted (both guest, nested-guest) for tracking
1670	PGM mode changes and AVX (for XCR0 syncing during worlds switching). */
1671	Assert(pVmcb->ctrl.u16InterceptWrCRx & RT_BIT(4));
1672
1673	/* Update VMCB with the shadow CR4 the appropriate VMCB clean bits. */
1674	Assert(!RT_HI_U32(uShadowCr4));
1675	pVmcb->guest.u64CR4 = uShadowCr4;
1676	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_CRX_EFER \| HMSVM_VMCB_CLEAN_INTERCEPTS);
1677
1678	return VINF_SUCCESS;
1679	}
1680
1681
1682	/**
1683	* Exports the guest (or nested-guest) control registers into the VMCB.
1684	*
1685	* @returns VBox status code.
1686	* @param pVCpu The cross context virtual CPU structure.
1687	* @param pVmcb Pointer to the VM control block.
1688	*
1689	* @remarks No-long-jump zone!!!
1690	*/
1691	static int hmR0SvmExportGuestControlRegs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1692	{
1693	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1694
1695	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR_MASK)
1696	{
1697	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR0)
1698	hmR0SvmExportGuestCR0(pVCpu, pVmcb);
1699
1700	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR2)
1701	{
1702	pVmcb->guest.u64CR2 = pVCpu->cpum.GstCtx.cr2;
1703	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CR2;
1704	}
1705
1706	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR3)
1707	hmR0SvmExportGuestCR3(pVCpu, pVmcb);
1708
1709	/* CR4 re-loading is ASSUMED to be done everytime we get in from ring-3! (XCR0) */
1710	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR4)
1711	{
1712	int rc = hmR0SvmExportGuestCR4(pVCpu, pVmcb);
1713	if (RT_FAILURE(rc))
1714	return rc;
1715	}
1716
1717	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_CR_MASK;
1718	}
1719	return VINF_SUCCESS;
1720	}
1721
1722
1723	/**
1724	* Exports the guest (or nested-guest) segment registers into the VMCB.
1725	*
1726	* @returns VBox status code.
1727	* @param pVCpu The cross context virtual CPU structure.
1728	* @param pVmcb Pointer to the VM control block.
1729	*
1730	* @remarks No-long-jump zone!!!
1731	*/
1732	static void hmR0SvmExportGuestSegmentRegs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1733	{
1734	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1735	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1736
1737	/* Guest segment registers. */
1738	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SREG_MASK)
1739	{
1740	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CS)
1741	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, CS, cs);
1742
1743	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SS)
1744	{
1745	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, SS, ss);
1746	pVmcb->guest.u8CPL = pCtx->ss.Attr.n.u2Dpl;
1747	}
1748
1749	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DS)
1750	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, DS, ds);
1751
1752	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_ES)
1753	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, ES, es);
1754
1755	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_FS)
1756	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, FS, fs);
1757
1758	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_GS)
1759	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, GS, gs);
1760
1761	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_SEG;
1762	}
1763
1764	/* Guest TR. */
1765	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_TR)
1766	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, TR, tr);
1767
1768	/* Guest LDTR. */
1769	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_LDTR)
1770	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, LDTR, ldtr);
1771
1772	/* Guest GDTR. */
1773	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_GDTR)
1774	{
1775	pVmcb->guest.GDTR.u32Limit = pCtx->gdtr.cbGdt;
1776	pVmcb->guest.GDTR.u64Base = pCtx->gdtr.pGdt;
1777	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1778	}
1779
1780	/* Guest IDTR. */
1781	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_IDTR)
1782	{
1783	pVmcb->guest.IDTR.u32Limit = pCtx->idtr.cbIdt;
1784	pVmcb->guest.IDTR.u64Base = pCtx->idtr.pIdt;
1785	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1786	}
1787
1788	pVCpu->hm.s.fCtxChanged &= ~( HM_CHANGED_GUEST_SREG_MASK
1789	\| HM_CHANGED_GUEST_TABLE_MASK);
1790	}
1791
1792
1793	/**
1794	* Exports the guest (or nested-guest) MSRs into the VMCB.
1795	*
1796	* @param pVCpu The cross context virtual CPU structure.
1797	* @param pVmcb Pointer to the VM control block.
1798	*
1799	* @remarks No-long-jump zone!!!
1800	*/
1801	static void hmR0SvmExportGuestMsrs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1802	{
1803	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1804	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1805
1806	/* Guest Sysenter MSRs. */
1807	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
1808	{
1809	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
1810	pVmcb->guest.u64SysEnterCS = pCtx->SysEnter.cs;
1811
1812	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
1813	pVmcb->guest.u64SysEnterEIP = pCtx->SysEnter.eip;
1814
1815	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
1816	pVmcb->guest.u64SysEnterESP = pCtx->SysEnter.esp;
1817	}
1818
1819	/*
1820	* Guest EFER MSR.
1821	* AMD-V requires guest EFER.SVME to be set. Weird.
1822	* See AMD spec. 15.5.1 "Basic Operation" \| "Canonicalization and Consistency Checks".
1823	*/
1824	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_EFER_MSR)
1825	{
1826	pVmcb->guest.u64EFER = pCtx->msrEFER \| MSR_K6_EFER_SVME;
1827	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1828	}
1829
1830	/* If the guest isn't in 64-bit mode, clear MSR_K6_LME bit, otherwise SVM expects amd64 shadow paging. */
1831	if ( !CPUMIsGuestInLongModeEx(pCtx)
1832	&& (pCtx->msrEFER & MSR_K6_EFER_LME))
1833	{
1834	pVmcb->guest.u64EFER &= ~MSR_K6_EFER_LME;
1835	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1836	}
1837
1838	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSCALL_MSRS)
1839	{
1840	pVmcb->guest.u64STAR = pCtx->msrSTAR;
1841	pVmcb->guest.u64LSTAR = pCtx->msrLSTAR;
1842	pVmcb->guest.u64CSTAR = pCtx->msrCSTAR;
1843	pVmcb->guest.u64SFMASK = pCtx->msrSFMASK;
1844	}
1845
1846	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_KERNEL_GS_BASE)
1847	pVmcb->guest.u64KernelGSBase = pCtx->msrKERNELGSBASE;
1848
1849	pVCpu->hm.s.fCtxChanged &= ~( HM_CHANGED_GUEST_SYSENTER_MSR_MASK
1850	\| HM_CHANGED_GUEST_EFER_MSR
1851	\| HM_CHANGED_GUEST_SYSCALL_MSRS
1852	\| HM_CHANGED_GUEST_KERNEL_GS_BASE);
1853
1854	/*
1855	* Setup the PAT MSR (applicable for Nested Paging only).
1856	*
1857	* The default value should be MSR_IA32_CR_PAT_INIT_VAL, but we treat all guest memory
1858	* as WB, so choose type 6 for all PAT slots, see @bugref{9634}.
1859	*
1860	* While guests can modify and see the modified values through the shadow values,
1861	* we shall not honor any guest modifications of this MSR to ensure caching is always
1862	* enabled similar to how we clear CR0.CD and NW bits.
1863	*
1864	* For nested-guests this needs to always be set as well, see @bugref{7243#c109}.
1865	*/
1866	pVmcb->guest.u64PAT = UINT64_C(0x0006060606060606);
1867
1868	/* Enable the last branch record bit if LBR virtualization is enabled. */
1869	if (pVmcb->ctrl.LbrVirt.n.u1LbrVirt)
1870	pVmcb->guest.u64DBGCTL = MSR_IA32_DEBUGCTL_LBR;
1871	}
1872
1873
1874	/**
1875	* Exports the guest (or nested-guest) debug state into the VMCB and programs
1876	* the necessary intercepts accordingly.
1877	*
1878	* @param pVCpu The cross context virtual CPU structure.
1879	* @param pVmcb Pointer to the VM control block.
1880	*
1881	* @remarks No-long-jump zone!!!
1882	* @remarks Requires EFLAGS to be up-to-date in the VMCB!
1883	*/
1884	static void hmR0SvmExportSharedDebugState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1885	{
1886	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1887
1888	/** @todo Figure out stepping with nested-guest. */
1889	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1890	{
1891	/*
1892	* We don't want to always intercept DRx read/writes for nested-guests as it causes
1893	* problems when the nested hypervisor isn't intercepting them, see @bugref{10080}.
1894	* Instead, they are strictly only requested when the nested hypervisor intercepts
1895	* them -- handled while merging VMCB controls.
1896	*
1897	* If neither the outer nor the nested-hypervisor is intercepting DRx read/writes,
1898	* then the nested-guest debug state should be actively loaded on the host so that
1899	* nested-guest reads/writes its own debug registers without causing VM-exits.
1900	*/
1901	if ( ( pVmcb->ctrl.u16InterceptRdDRx != 0xffff
1902	\|\| pVmcb->ctrl.u16InterceptWrDRx != 0xffff)
1903	&& !CPUMIsGuestDebugStateActive(pVCpu))
1904	{
1905	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
1906	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
1907	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
1908	Assert(CPUMIsGuestDebugStateActive(pVCpu));
1909	}
1910
1911	pVmcb->guest.u64DR6 = pCtx->dr[6];
1912	pVmcb->guest.u64DR7 = pCtx->dr[7];
1913	return;
1914	}
1915
1916	/*
1917	* Anyone single stepping on the host side? If so, we'll have to use the
1918	* trap flag in the guest EFLAGS since AMD-V doesn't have a trap flag on
1919	* the VMM level like the VT-x implementations does.
1920	*/
1921	bool fInterceptMovDRx = false;
1922	bool const fStepping = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
1923	if (fStepping)
1924	{
1925	pVCpu->hmr0.s.fClearTrapFlag = true;
1926	pVmcb->guest.u64RFlags \|= X86_EFL_TF;
1927	fInterceptMovDRx = true; /* Need clean DR6, no guest mess. */
1928	}
1929
1930	if ( fStepping
1931	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
1932	{
1933	/*
1934	* Use the combined guest and host DRx values found in the hypervisor
1935	* register set because the debugger has breakpoints active or someone
1936	* is single stepping on the host side.
1937	*
1938	* Note! DBGF expects a clean DR6 state before executing guest code.
1939	*/
1940	if (!CPUMIsHyperDebugStateActive(pVCpu))
1941	{
1942	CPUMR0LoadHyperDebugState(pVCpu, false /* include DR6 */);
1943	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
1944	Assert(CPUMIsHyperDebugStateActive(pVCpu));
1945	}
1946
1947	/* Update DR6 & DR7. (The other DRx values are handled by CPUM one way or the other.) */
1948	if ( pVmcb->guest.u64DR6 != X86_DR6_INIT_VAL
1949	\|\| pVmcb->guest.u64DR7 != CPUMGetHyperDR7(pVCpu))
1950	{
1951	pVmcb->guest.u64DR7 = CPUMGetHyperDR7(pVCpu);
1952	pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
1953	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1954	}
1955
1956	/** @todo If we cared, we could optimize to allow the guest to read registers
1957	* with the same values. */
1958	fInterceptMovDRx = true;
1959	pVCpu->hmr0.s.fUsingHyperDR7 = true;
1960	Log5(("hmR0SvmExportSharedDebugState: Loaded hyper DRx\n"));
1961	}
1962	else
1963	{
1964	/*
1965	* Update DR6, DR7 with the guest values if necessary.
1966	*/
1967	if ( pVmcb->guest.u64DR7 != pCtx->dr[7]
1968	\|\| pVmcb->guest.u64DR6 != pCtx->dr[6])
1969	{
1970	pVmcb->guest.u64DR7 = pCtx->dr[7];
1971	pVmcb->guest.u64DR6 = pCtx->dr[6];
1972	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1973	}
1974	pVCpu->hmr0.s.fUsingHyperDR7 = false;
1975
1976	/*
1977	* If the guest has enabled debug registers, we need to load them prior to
1978	* executing guest code so they'll trigger at the right time.
1979	*/
1980	if (pCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
1981	{
1982	if (!CPUMIsGuestDebugStateActive(pVCpu))
1983	{
1984	CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
1985	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
1986	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
1987	Assert(CPUMIsGuestDebugStateActive(pVCpu));
1988	}
1989	Log5(("hmR0SvmExportSharedDebugState: Loaded guest DRx\n"));
1990	}
1991	/*
1992	* If no debugging enabled, we'll lazy load DR0-3. We don't need to
1993	* intercept #DB as DR6 is updated in the VMCB.
1994	*
1995	* Note! If we cared and dared, we could skip intercepting \#DB here.
1996	* However, \#DB shouldn't be performance critical, so we'll play safe
1997	* and keep the code similar to the VT-x code and always intercept it.
1998	*/
1999	else if (!CPUMIsGuestDebugStateActive(pVCpu))
2000	fInterceptMovDRx = true;
2001	}
2002
2003	Assert(pVmcb->ctrl.u32InterceptXcpt & RT_BIT_32(X86_XCPT_DB));
2004	if (fInterceptMovDRx)
2005	{
2006	if ( pVmcb->ctrl.u16InterceptRdDRx != 0xffff
2007	\|\| pVmcb->ctrl.u16InterceptWrDRx != 0xffff)
2008	{
2009	pVmcb->ctrl.u16InterceptRdDRx = 0xffff;
2010	pVmcb->ctrl.u16InterceptWrDRx = 0xffff;
2011	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2012	}
2013	}
2014	else
2015	{
2016	if ( pVmcb->ctrl.u16InterceptRdDRx
2017	\|\| pVmcb->ctrl.u16InterceptWrDRx)
2018	{
2019	pVmcb->ctrl.u16InterceptRdDRx = 0;
2020	pVmcb->ctrl.u16InterceptWrDRx = 0;
2021	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2022	}
2023	}
2024	Log4Func(("DR6=%#RX64 DR7=%#RX64\n", pCtx->dr[6], pCtx->dr[7]));
2025	}
2026
2027	/**
2028	* Exports the hardware virtualization state into the nested-guest
2029	* VMCB.
2030	*
2031	* @param pVCpu The cross context virtual CPU structure.
2032	* @param pVmcb Pointer to the VM control block.
2033	*
2034	* @remarks No-long-jump zone!!!
2035	*/
2036	static void hmR0SvmExportGuestHwvirtState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2037	{
2038	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2039
2040	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_HWVIRT)
2041	{
2042	if (pVmcb->ctrl.IntCtrl.n.u1VGifEnable)
2043	{
2044	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2045	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
2046
2047	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx); /* Nested VGIF is not supported yet. */
2048	Assert(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VGIF); /* Physical hardware supports VGIF. */
2049	Assert(HMIsSvmVGifActive(pVM)); /* Outer VM has enabled VGIF. */
2050	NOREF(pVM);
2051
2052	pVmcb->ctrl.IntCtrl.n.u1VGif = CPUMGetGuestGif(pCtx);
2053	}
2054
2055	/*
2056	* Ensure the nested-guest pause-filter counters don't exceed the outer guest values esp.
2057	* since SVM doesn't have a preemption timer.
2058	*
2059	* We do this here rather than in hmR0SvmSetupVmcbNested() as we may have been executing the
2060	* nested-guest in IEM incl. PAUSE instructions which would update the pause-filter counters
2061	* and may continue execution in SVM R0 without a nested-guest #VMEXIT in between.
2062	*/
2063	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2064	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
2065	uint16_t const uGuestPauseFilterCount = pVM->hm.s.svm.cPauseFilter;
2066	uint16_t const uGuestPauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
2067	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, SVM_CTRL_INTERCEPT_PAUSE))
2068	{
2069	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2070	pVmcbCtrl->u16PauseFilterCount = RT_MIN(pCtx->hwvirt.svm.cPauseFilter, uGuestPauseFilterCount);
2071	pVmcbCtrl->u16PauseFilterThreshold = RT_MIN(pCtx->hwvirt.svm.cPauseFilterThreshold, uGuestPauseFilterThreshold);
2072	}
2073	else
2074	{
2075	/** @todo r=ramshankar: We can turn these assignments into assertions. */
2076	pVmcbCtrl->u16PauseFilterCount = uGuestPauseFilterCount;
2077	pVmcbCtrl->u16PauseFilterThreshold = uGuestPauseFilterThreshold;
2078	}
2079	pVmcbCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2080
2081	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_HWVIRT;
2082	}
2083	}
2084
2085
2086	/**
2087	* Exports the guest APIC TPR state into the VMCB.
2088	*
2089	* @returns VBox status code.
2090	* @param pVCpu The cross context virtual CPU structure.
2091	* @param pVmcb Pointer to the VM control block.
2092	*/
2093	static int hmR0SvmExportGuestApicTpr(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2094	{
2095	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2096
2097	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
2098	{
2099	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2100	if ( PDMHasApic(pVM)
2101	&& APICIsEnabled(pVCpu))
2102	{
2103	bool fPendingIntr;
2104	uint8_t u8Tpr;
2105	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, NULL /* pu8PendingIrq */);
2106	AssertRCReturn(rc, rc);
2107
2108	/* Assume that we need to trap all TPR accesses and thus need not check on
2109	every #VMEXIT if we should update the TPR. */
2110	Assert(pVmcb->ctrl.IntCtrl.n.u1VIntrMasking);
2111	pVCpu->hmr0.s.svm.fSyncVTpr = false;
2112
2113	if (!pVM->hm.s.fTprPatchingActive)
2114	{
2115	/* Bits 3-0 of the VTPR field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2116	pVmcb->ctrl.IntCtrl.n.u8VTPR = (u8Tpr >> 4);
2117
2118	/* If there are interrupts pending, intercept CR8 writes to evaluate ASAP if we
2119	can deliver the interrupt to the guest. */
2120	if (fPendingIntr)
2121	pVmcb->ctrl.u16InterceptWrCRx \|= RT_BIT(8);
2122	else
2123	{
2124	pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(8);
2125	pVCpu->hmr0.s.svm.fSyncVTpr = true;
2126	}
2127
2128	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_INTERCEPTS \| HMSVM_VMCB_CLEAN_INT_CTRL);
2129	}
2130	else
2131	{
2132	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2133	pVmcb->guest.u64LSTAR = u8Tpr;
2134	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hmr0.s.svm.pvMsrBitmap;
2135
2136	/* If there are interrupts pending, intercept LSTAR writes, otherwise don't intercept reads or writes. */
2137	if (fPendingIntr)
2138	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_INTERCEPT_WRITE);
2139	else
2140	{
2141	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
2142	pVCpu->hmr0.s.svm.fSyncVTpr = true;
2143	}
2144	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
2145	}
2146	}
2147	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
2148	}
2149	return VINF_SUCCESS;
2150	}
2151
2152
2153	/**
2154	* Sets up the exception interrupts required for guest execution in the VMCB.
2155	*
2156	* @param pVCpu The cross context virtual CPU structure.
2157	* @param pVmcb Pointer to the VM control block.
2158	*
2159	* @remarks No-long-jump zone!!!
2160	*/
2161	static void hmR0SvmExportGuestXcptIntercepts(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2162	{
2163	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2164
2165	/* If we modify intercepts from here, please check & adjust hmR0SvmMergeVmcbCtrlsNested() if required. */
2166	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_SVM_XCPT_INTERCEPTS)
2167	{
2168	/* Trap #UD for GIM provider (e.g. for hypercalls). */
2169	if (pVCpu->hm.s.fGIMTrapXcptUD \|\| pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit)
2170	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_UD);
2171	else
2172	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_UD);
2173
2174	/* Trap #BP for INT3 debug breakpoints set by the VM debugger. */
2175	if (pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
2176	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_BP);
2177	else
2178	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_BP);
2179
2180	/* The remaining intercepts are handled elsewhere, e.g. in hmR0SvmExportGuestCR0(). */
2181	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_SVM_XCPT_INTERCEPTS);
2182	}
2183	}
2184
2185
2186	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2187	/**
2188	* Merges guest and nested-guest intercepts for executing the nested-guest using
2189	* hardware-assisted SVM.
2190	*
2191	* This merges the guest and nested-guest intercepts in a way that if the outer
2192	* guest intercept is set we need to intercept it in the nested-guest as
2193	* well.
2194	*
2195	* @param pVCpu The cross context virtual CPU structure.
2196	* @param pVmcbNstGst Pointer to the nested-guest VM control block.
2197	*/
2198	static void hmR0SvmMergeVmcbCtrlsNested(PVMCPUCC pVCpu)
2199	{
2200	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2201	PCSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
2202	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2203	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2204
2205	/* Merge the guest's CR intercepts into the nested-guest VMCB. */
2206	pVmcbNstGstCtrl->u16InterceptRdCRx \|= pVmcb->ctrl.u16InterceptRdCRx;
2207	pVmcbNstGstCtrl->u16InterceptWrCRx \|= pVmcb->ctrl.u16InterceptWrCRx;
2208
2209	/* Always intercept CR4 writes for tracking PGM mode changes and AVX (for
2210	XCR0 syncing during worlds switching). */
2211	pVmcbNstGstCtrl->u16InterceptWrCRx \|= RT_BIT(4);
2212
2213	/* Without nested paging, intercept CR3 reads and writes as we load shadow page tables. */
2214	if (!pVM->hmr0.s.fNestedPaging)
2215	{
2216	pVmcbNstGstCtrl->u16InterceptRdCRx \|= RT_BIT(3);
2217	pVmcbNstGstCtrl->u16InterceptWrCRx \|= RT_BIT(3);
2218	}
2219
2220	/* Merge the guest's DR intercepts into the nested-guest VMCB. */
2221	pVmcbNstGstCtrl->u16InterceptRdDRx \|= pVmcb->ctrl.u16InterceptRdDRx;
2222	pVmcbNstGstCtrl->u16InterceptWrDRx \|= pVmcb->ctrl.u16InterceptWrDRx;
2223
2224	/*
2225	* Merge the guest's exception intercepts into the nested-guest VMCB.
2226	*
2227	* - #UD: Exclude these as the outer guest's GIM hypercalls are not applicable
2228	* while executing the nested-guest.
2229	*
2230	* - #BP: Exclude breakpoints set by the VM debugger for the outer guest. This can
2231	* be tweaked later depending on how we wish to implement breakpoints.
2232	*
2233	* - #GP: Exclude these as it's the inner VMMs problem to get vmsvga 3d drivers
2234	* loaded into their guests, not ours.
2235	*
2236	* Warning!! This ASSUMES we only intercept \#UD for hypercall purposes and \#BP
2237	* for VM debugger breakpoints, see hmR0SvmExportGuestXcptIntercepts().
2238	*/
2239	#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
2240	pVmcbNstGstCtrl->u32InterceptXcpt \|= pVmcb->ctrl.u32InterceptXcpt
2241	& ~( RT_BIT(X86_XCPT_UD)
2242	\| RT_BIT(X86_XCPT_BP)
2243	\| (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv ? RT_BIT(X86_XCPT_GP) : 0));
2244	#else
2245	pVmcbNstGstCtrl->u32InterceptXcpt \|= pVmcb->ctrl.u32InterceptXcpt;
2246	#endif
2247
2248	/*
2249	* Adjust intercepts while executing the nested-guest that differ from the
2250	* outer guest intercepts.
2251	*
2252	* - VINTR: Exclude the outer guest intercept as we don't need to cause VINTR #VMEXITs
2253	* that belong to the nested-guest to the outer guest.
2254	*
2255	* - VMMCALL: Exclude the outer guest intercept as when it's also not intercepted by
2256	* the nested-guest, the physical CPU raises a \#UD exception as expected.
2257	*/
2258	pVmcbNstGstCtrl->u64InterceptCtrl \|= (pVmcb->ctrl.u64InterceptCtrl & ~( SVM_CTRL_INTERCEPT_VINTR
2259	\| SVM_CTRL_INTERCEPT_VMMCALL))
2260	\| HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS;
2261
2262	Assert( (pVmcbNstGstCtrl->u64InterceptCtrl & HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS)
2263	== HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS);
2264
2265	/* Finally, update the VMCB clean bits. */
2266	pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2267	}
2268	#endif
2269
2270
2271	/**
2272	* Enters the AMD-V session.
2273	*
2274	* @returns VBox status code.
2275	* @param pVCpu The cross context virtual CPU structure.
2276	*/
2277	VMMR0DECL(int) SVMR0Enter(PVMCPUCC pVCpu)
2278	{
2279	AssertPtr(pVCpu);
2280	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fSupported);
2281	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2282
2283	LogFlowFunc(("pVCpu=%p\n", pVCpu));
2284	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
2285	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
2286
2287	pVCpu->hmr0.s.fLeaveDone = false;
2288	return VINF_SUCCESS;
2289	}
2290
2291
2292	/**
2293	* Thread-context callback for AMD-V.
2294	*
2295	* This is used together with RTThreadCtxHookCreate() on platforms which
2296	* supports it, and directly from VMMR0EmtPrepareForBlocking() and
2297	* VMMR0EmtResumeAfterBlocking() on platforms which don't.
2298	*
2299	* @param enmEvent The thread-context event.
2300	* @param pVCpu The cross context virtual CPU structure.
2301	* @param fGlobalInit Whether global VT-x/AMD-V init. is used.
2302	* @thread EMT(pVCpu)
2303	*/
2304	VMMR0DECL(void) SVMR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPUCC pVCpu, bool fGlobalInit)
2305	{
2306	NOREF(fGlobalInit);
2307
2308	switch (enmEvent)
2309	{
2310	case RTTHREADCTXEVENT_OUT:
2311	{
2312	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2313	VMCPU_ASSERT_EMT(pVCpu);
2314
2315	/* No longjmps (log-flush, locks) in this fragile context. */
2316	VMMRZCallRing3Disable(pVCpu);
2317
2318	if (!pVCpu->hmr0.s.fLeaveDone)
2319	{
2320	hmR0SvmLeave(pVCpu, false /* fImportState */);
2321	pVCpu->hmr0.s.fLeaveDone = true;
2322	}
2323
2324	/* Leave HM context, takes care of local init (term). */
2325	int rc = HMR0LeaveCpu(pVCpu);
2326	AssertRC(rc); NOREF(rc);
2327
2328	/* Restore longjmp state. */
2329	VMMRZCallRing3Enable(pVCpu);
2330	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
2331	break;
2332	}
2333
2334	case RTTHREADCTXEVENT_IN:
2335	{
2336	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2337	VMCPU_ASSERT_EMT(pVCpu);
2338
2339	/* No longjmps (log-flush, locks) in this fragile context. */
2340	VMMRZCallRing3Disable(pVCpu);
2341
2342	/*
2343	* Initialize the bare minimum state required for HM. This takes care of
2344	* initializing AMD-V if necessary (onlined CPUs, local init etc.)
2345	*/
2346	int rc = hmR0EnterCpu(pVCpu);
2347	AssertRC(rc); NOREF(rc);
2348	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
2349	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
2350
2351	pVCpu->hmr0.s.fLeaveDone = false;
2352
2353	/* Restore longjmp state. */
2354	VMMRZCallRing3Enable(pVCpu);
2355	break;
2356	}
2357
2358	default:
2359	break;
2360	}
2361	}
2362
2363
2364	/**
2365	* Saves the host state.
2366	*
2367	* @returns VBox status code.
2368	* @param pVCpu The cross context virtual CPU structure.
2369	*
2370	* @remarks No-long-jump zone!!!
2371	*/
2372	VMMR0DECL(int) SVMR0ExportHostState(PVMCPUCC pVCpu)
2373	{
2374	NOREF(pVCpu);
2375
2376	/* Nothing to do here. AMD-V does this for us automatically during the world-switch. */
2377	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_HOST_CONTEXT);
2378	return VINF_SUCCESS;
2379	}
2380
2381
2382	/**
2383	* Exports the guest or nested-guest state from the virtual-CPU context into the
2384	* VMCB.
2385	*
2386	* Also sets up the appropriate VMRUN function to execute guest or nested-guest
2387	* code based on the virtual-CPU mode.
2388	*
2389	* @returns VBox status code.
2390	* @param pVCpu The cross context virtual CPU structure.
2391	* @param pSvmTransient Pointer to the SVM-transient structure.
2392	*
2393	* @remarks No-long-jump zone!!!
2394	*/
2395	static int hmR0SvmExportGuestState(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient)
2396	{
2397	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
2398
2399	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
2400	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2401	Assert(pVmcb);
2402
2403	pVmcb->guest.u64RIP = pCtx->rip;
2404	pVmcb->guest.u64RSP = pCtx->rsp;
2405	pVmcb->guest.u64RFlags = pCtx->eflags.u32;
2406	pVmcb->guest.u64RAX = pCtx->rax;
2407
2408	bool const fIsNestedGuest = pSvmTransient->fIsNestedGuest;
2409	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
2410
2411	int rc = hmR0SvmExportGuestControlRegs(pVCpu, pVmcb);
2412	AssertRCReturnStmt(rc, ASMSetFlags(fEFlags), rc);
2413	hmR0SvmExportGuestSegmentRegs(pVCpu, pVmcb);
2414	hmR0SvmExportGuestMsrs(pVCpu, pVmcb);
2415	hmR0SvmExportGuestHwvirtState(pVCpu, pVmcb);
2416
2417	ASMSetFlags(fEFlags);
2418
2419	if (!fIsNestedGuest)
2420	{
2421	/* hmR0SvmExportGuestApicTpr() must be called -after- hmR0SvmExportGuestMsrs() as we
2422	otherwise we would overwrite the LSTAR MSR that we use for TPR patching. */
2423	hmR0SvmExportGuestApicTpr(pVCpu, pVmcb);
2424	hmR0SvmExportGuestXcptIntercepts(pVCpu, pVmcb);
2425	}
2426
2427	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
2428	uint64_t fUnusedMask = HM_CHANGED_GUEST_RIP
2429	\| HM_CHANGED_GUEST_RFLAGS
2430	\| HM_CHANGED_GUEST_GPRS_MASK
2431	\| HM_CHANGED_GUEST_X87
2432	\| HM_CHANGED_GUEST_SSE_AVX
2433	\| HM_CHANGED_GUEST_OTHER_XSAVE
2434	\| HM_CHANGED_GUEST_XCRx
2435	\| HM_CHANGED_GUEST_TSC_AUX
2436	\| HM_CHANGED_GUEST_OTHER_MSRS;
2437	if (fIsNestedGuest)
2438	fUnusedMask \|= HM_CHANGED_SVM_XCPT_INTERCEPTS
2439	\| HM_CHANGED_GUEST_APIC_TPR;
2440
2441	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( fUnusedMask
2442	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_SVM_MASK)));
2443
2444	#ifdef VBOX_STRICT
2445	/*
2446	* All of the guest-CPU state and SVM keeper bits should be exported here by now,
2447	* except for the host-context and/or shared host-guest context bits.
2448	*/
2449	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
2450	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)),
2451	("fCtxChanged=%#RX64\n", fCtxChanged));
2452
2453	/*
2454	* If we need to log state that isn't always imported, we'll need to import them here.
2455	* See hmR0SvmPostRunGuest() for which part of the state is imported uncondtionally.
2456	*/
2457	hmR0SvmLogState(pVCpu, pVmcb, "hmR0SvmExportGuestState", 0 /* fFlags /, 0 / uVerbose */);
2458	#endif
2459
2460	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
2461	return VINF_SUCCESS;
2462	}
2463
2464	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2465
2466	/**
2467	* Merges the guest and nested-guest MSR permission bitmap.
2468	*
2469	* If the guest is intercepting an MSR we need to intercept it regardless of
2470	* whether the nested-guest is intercepting it or not.
2471	*
2472	* @param pHostCpu The HM physical-CPU structure.
2473	* @param pVCpu The cross context virtual CPU structure.
2474	*
2475	* @remarks No-long-jmp zone!!!
2476	*/
2477	DECLINLINE(void) hmR0SvmMergeMsrpmNested(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu)
2478	{
2479	uint64_t const pu64GstMsrpm = (uint64_t const )pVCpu->hmr0.s.svm.pvMsrBitmap;
2480	uint64_t const pu64NstGstMsrpm = (uint64_t const )&pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[0];
2481	uint64_t pu64DstMsrpm = (uint64_t )pHostCpu->n.svm.pvNstGstMsrpm;
2482
2483	/* MSRPM bytes from offset 0x1800 are reserved, so we stop merging there. */
2484	uint32_t const offRsvdQwords = 0x1800 >> 3;
2485	for (uint32_t i = 0; i < offRsvdQwords; i++)
2486	pu64DstMsrpm[i] = pu64NstGstMsrpm[i] \| pu64GstMsrpm[i];
2487	}
2488
2489
2490	/**
2491	* Caches the nested-guest VMCB fields before we modify them for execution using
2492	* hardware-assisted SVM.
2493	*
2494	* @returns true if the VMCB was previously already cached, false otherwise.
2495	* @param pVCpu The cross context virtual CPU structure.
2496	*
2497	* @sa HMNotifySvmNstGstVmexit.
2498	*/
2499	static bool hmR0SvmCacheVmcbNested(PVMCPUCC pVCpu)
2500	{
2501	/*
2502	* Cache the nested-guest programmed VMCB fields if we have not cached it yet.
2503	* Otherwise we risk re-caching the values we may have modified, see @bugref{7243#c44}.
2504	*
2505	* Nested-paging CR3 is not saved back into the VMCB on #VMEXIT, hence no need to
2506	* cache and restore it, see AMD spec. 15.25.4 "Nested Paging and VMRUN/#VMEXIT".
2507	*/
2508	PSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
2509	bool const fWasCached = pVmcbNstGstCache->fCacheValid;
2510	if (!fWasCached)
2511	{
2512	PCSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2513	PCSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2514	pVmcbNstGstCache->u16InterceptRdCRx = pVmcbNstGstCtrl->u16InterceptRdCRx;
2515	pVmcbNstGstCache->u16InterceptWrCRx = pVmcbNstGstCtrl->u16InterceptWrCRx;
2516	pVmcbNstGstCache->u16InterceptRdDRx = pVmcbNstGstCtrl->u16InterceptRdDRx;
2517	pVmcbNstGstCache->u16InterceptWrDRx = pVmcbNstGstCtrl->u16InterceptWrDRx;
2518	pVmcbNstGstCache->u16PauseFilterThreshold = pVmcbNstGstCtrl->u16PauseFilterThreshold;
2519	pVmcbNstGstCache->u16PauseFilterCount = pVmcbNstGstCtrl->u16PauseFilterCount;
2520	pVmcbNstGstCache->u32InterceptXcpt = pVmcbNstGstCtrl->u32InterceptXcpt;
2521	pVmcbNstGstCache->u64InterceptCtrl = pVmcbNstGstCtrl->u64InterceptCtrl;
2522	pVmcbNstGstCache->u64TSCOffset = pVmcbNstGstCtrl->u64TSCOffset;
2523	pVmcbNstGstCache->fVIntrMasking = pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking;
2524	pVmcbNstGstCache->fNestedPaging = pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging;
2525	pVmcbNstGstCache->fLbrVirt = pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt;
2526	pVmcbNstGstCache->fCacheValid = true;
2527	Log4Func(("Cached VMCB fields\n"));
2528	}
2529
2530	return fWasCached;
2531	}
2532
2533
2534	/**
2535	* Sets up the nested-guest VMCB for execution using hardware-assisted SVM.
2536	*
2537	* This is done the first time we enter nested-guest execution using SVM R0
2538	* until the nested-guest \#VMEXIT (not to be confused with physical CPU
2539	* \#VMEXITs which may or may not cause a corresponding nested-guest \#VMEXIT).
2540	*
2541	* @param pVCpu The cross context virtual CPU structure.
2542	*/
2543	static void hmR0SvmSetupVmcbNested(PVMCPUCC pVCpu)
2544	{
2545	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2546	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2547
2548	HMSVM_ASSERT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2549
2550	/*
2551	* First cache the nested-guest VMCB fields we may potentially modify.
2552	*/
2553	bool const fVmcbCached = hmR0SvmCacheVmcbNested(pVCpu);
2554	if (!fVmcbCached)
2555	{
2556	/*
2557	* The IOPM of the nested-guest can be ignored because the the guest always
2558	* intercepts all IO port accesses. Thus, we'll swap to the guest IOPM rather
2559	* than the nested-guest IOPM and swap the field back on the #VMEXIT.
2560	*/
2561	pVmcbNstGstCtrl->u64IOPMPhysAddr = g_HCPhysIOBitmap;
2562
2563	/*
2564	* Use the same nested-paging as the outer guest. We can't dynamically switch off
2565	* nested-paging suddenly while executing a VM (see assertion at the end of
2566	* Trap0eHandler() in PGMAllBth.h).
2567	*/
2568	pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging = pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging;
2569
2570	/* Always enable V_INTR_MASKING as we do not want to allow access to the physical APIC TPR. */
2571	pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking = 1;
2572
2573	/*
2574	* Turn off TPR syncing on #VMEXIT for nested-guests as CR8 intercepts are subject
2575	* to the nested-guest intercepts and we always run with V_INTR_MASKING.
2576	*/
2577	pVCpu->hmr0.s.svm.fSyncVTpr = false;
2578
2579	# ifdef DEBUG_ramshankar
2580	/* For debugging purposes - copy the LBR info. from outer guest VMCB. */
2581	pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt = pVmcb->ctrl.LbrVirt.n.u1LbrVirt;
2582	# endif
2583
2584	/*
2585	* If we don't expose Virtualized-VMSAVE/VMLOAD feature to the outer guest, we
2586	* need to intercept VMSAVE/VMLOAD instructions executed by the nested-guest.
2587	*/
2588	if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVirtVmsaveVmload)
2589	pVmcbNstGstCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_VMSAVE
2590	\| SVM_CTRL_INTERCEPT_VMLOAD;
2591
2592	/*
2593	* If we don't expose Virtual GIF feature to the outer guest, we need to intercept
2594	* CLGI/STGI instructions executed by the nested-guest.
2595	*/
2596	if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVGif)
2597	pVmcbNstGstCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_CLGI
2598	\| SVM_CTRL_INTERCEPT_STGI;
2599
2600	/* Merge the guest and nested-guest intercepts. */
2601	hmR0SvmMergeVmcbCtrlsNested(pVCpu);
2602
2603	/* Update the VMCB clean bits. */
2604	pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2605	}
2606	else
2607	{
2608	Assert(!pVCpu->hmr0.s.svm.fSyncVTpr);
2609	Assert(pVmcbNstGstCtrl->u64IOPMPhysAddr == g_HCPhysIOBitmap);
2610	Assert(RT_BOOL(pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging) == pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
2611	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPagingCfg == pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
2612	}
2613	}
2614
2615	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
2616
2617	/**
2618	* Exports the state shared between the host and guest (or nested-guest) into
2619	* the VMCB.
2620	*
2621	* @param pVCpu The cross context virtual CPU structure.
2622	* @param pVmcb Pointer to the VM control block.
2623	*
2624	* @remarks No-long-jump zone!!!
2625	*/
2626	static void hmR0SvmExportSharedState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2627	{
2628	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2629	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2630
2631	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
2632	hmR0SvmExportSharedDebugState(pVCpu, pVmcb);
2633
2634	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
2635	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE),
2636	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
2637	}
2638
2639
2640	/**
2641	* Worker for SVMR0ImportStateOnDemand.
2642	*
2643	* @param pVCpu The cross context virtual CPU structure.
2644	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
2645	*/
2646	static void hmR0SvmImportGuestState(PVMCPUCC pVCpu, uint64_t fWhat)
2647	{
2648	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
2649
2650	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2651	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
2652	PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
2653	PCSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
2654
2655	/*
2656	* We disable interrupts to make the updating of the state and in particular
2657	* the fExtrn modification atomic wrt to preemption hooks.
2658	*/
2659	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
2660
2661	fWhat &= pCtx->fExtrn;
2662	if (fWhat)
2663	{
2664	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2665	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
2666	{
2667	if (pVmcbCtrl->IntCtrl.n.u1VGifEnable)
2668	{
2669	Assert(!CPUMIsGuestInSvmNestedHwVirtMode(pCtx)); /* We don't yet support passing VGIF feature to the guest. */
2670	Assert(HMIsSvmVGifActive(pVCpu->CTX_SUFF(pVM))); /* VM has configured it. */
2671	CPUMSetGuestGif(pCtx, pVmcbCtrl->IntCtrl.n.u1VGif);
2672	}
2673	}
2674
2675	if (fWhat & CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ)
2676	{
2677	if ( !pVmcbCtrl->IntCtrl.n.u1VIrqPending
2678	&& VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST))
2679	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST);
2680	}
2681	#endif
2682
2683	if (fWhat & CPUMCTX_EXTRN_INHIBIT_INT)
2684	{
2685	if (pVmcbCtrl->IntShadow.n.u1IntShadow)
2686	EMSetInhibitInterruptsPC(pVCpu, pVmcbGuest->u64RIP);
2687	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2688	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2689	}
2690
2691	if (fWhat & CPUMCTX_EXTRN_RIP)
2692	pCtx->rip = pVmcbGuest->u64RIP;
2693
2694	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
2695	{
2696	pCtx->eflags.u32 = pVmcbGuest->u64RFlags;
2697	if (pVCpu->hmr0.s.fClearTrapFlag)
2698	{
2699	pVCpu->hmr0.s.fClearTrapFlag = false;
2700	pCtx->eflags.Bits.u1TF = 0;
2701	}
2702	}
2703
2704	if (fWhat & CPUMCTX_EXTRN_RSP)
2705	pCtx->rsp = pVmcbGuest->u64RSP;
2706
2707	if (fWhat & CPUMCTX_EXTRN_RAX)
2708	pCtx->rax = pVmcbGuest->u64RAX;
2709
2710	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
2711	{
2712	if (fWhat & CPUMCTX_EXTRN_CS)
2713	{
2714	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, CS, cs);
2715	/* Correct the CS granularity bit. Haven't seen it being wrong in any other register (yet). */
2716	/** @todo SELM might need to be fixed as it too should not care about the
2717	* granularity bit. See @bugref{6785}. */
2718	if ( !pCtx->cs.Attr.n.u1Granularity
2719	&& pCtx->cs.Attr.n.u1Present
2720	&& pCtx->cs.u32Limit > UINT32_C(0xfffff))
2721	{
2722	Assert((pCtx->cs.u32Limit & 0xfff) == 0xfff);
2723	pCtx->cs.Attr.n.u1Granularity = 1;
2724	}
2725	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, cs);
2726	}
2727	if (fWhat & CPUMCTX_EXTRN_SS)
2728	{
2729	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, SS, ss);
2730	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, ss);
2731	/*
2732	* Sync the hidden SS DPL field. AMD CPUs have a separate CPL field in the
2733	* VMCB and uses that and thus it's possible that when the CPL changes during
2734	* guest execution that the SS DPL isn't updated by AMD-V. Observed on some
2735	* AMD Fusion CPUs with 64-bit guests.
2736	*
2737	* See AMD spec. 15.5.1 "Basic operation".
2738	*/
2739	Assert(!(pVmcbGuest->u8CPL & ~0x3));
2740	uint8_t const uCpl = pVmcbGuest->u8CPL;
2741	if (pCtx->ss.Attr.n.u2Dpl != uCpl)
2742	pCtx->ss.Attr.n.u2Dpl = uCpl & 0x3;
2743	}
2744	if (fWhat & CPUMCTX_EXTRN_DS)
2745	{
2746	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, DS, ds);
2747	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, ds);
2748	}
2749	if (fWhat & CPUMCTX_EXTRN_ES)
2750	{
2751	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, ES, es);
2752	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, es);
2753	}
2754	if (fWhat & CPUMCTX_EXTRN_FS)
2755	{
2756	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, FS, fs);
2757	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, fs);
2758	}
2759	if (fWhat & CPUMCTX_EXTRN_GS)
2760	{
2761	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, GS, gs);
2762	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, gs);
2763	}
2764	}
2765
2766	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
2767	{
2768	if (fWhat & CPUMCTX_EXTRN_TR)
2769	{
2770	/*
2771	* Fixup TR attributes so it's compatible with Intel. Important when saved-states
2772	* are used between Intel and AMD, see @bugref{6208#c39}.
2773	* ASSUME that it's normally correct and that we're in 32-bit or 64-bit mode.
2774	*/
2775	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, TR, tr);
2776	if (pCtx->tr.Attr.n.u4Type != X86_SEL_TYPE_SYS_386_TSS_BUSY)
2777	{
2778	if ( pCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL
2779	\|\| CPUMIsGuestInLongModeEx(pCtx))
2780	pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
2781	else if (pCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
2782	pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_286_TSS_BUSY;
2783	}
2784	}
2785
2786	if (fWhat & CPUMCTX_EXTRN_LDTR)
2787	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, LDTR, ldtr);
2788
2789	if (fWhat & CPUMCTX_EXTRN_GDTR)
2790	{
2791	pCtx->gdtr.cbGdt = pVmcbGuest->GDTR.u32Limit;
2792	pCtx->gdtr.pGdt = pVmcbGuest->GDTR.u64Base;
2793	}
2794
2795	if (fWhat & CPUMCTX_EXTRN_IDTR)
2796	{
2797	pCtx->idtr.cbIdt = pVmcbGuest->IDTR.u32Limit;
2798	pCtx->idtr.pIdt = pVmcbGuest->IDTR.u64Base;
2799	}
2800	}
2801
2802	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
2803	{
2804	pCtx->msrSTAR = pVmcbGuest->u64STAR;
2805	pCtx->msrLSTAR = pVmcbGuest->u64LSTAR;
2806	pCtx->msrCSTAR = pVmcbGuest->u64CSTAR;
2807	pCtx->msrSFMASK = pVmcbGuest->u64SFMASK;
2808	}
2809
2810	if ( (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
2811	&& !pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit /* Intercepted. AMD-V would clear the high 32 bits of EIP & ESP. */)
2812	{
2813	pCtx->SysEnter.cs = pVmcbGuest->u64SysEnterCS;
2814	pCtx->SysEnter.eip = pVmcbGuest->u64SysEnterEIP;
2815	pCtx->SysEnter.esp = pVmcbGuest->u64SysEnterESP;
2816	}
2817
2818	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
2819	pCtx->msrKERNELGSBASE = pVmcbGuest->u64KernelGSBase;
2820
2821	if (fWhat & CPUMCTX_EXTRN_DR_MASK)
2822	{
2823	if (fWhat & CPUMCTX_EXTRN_DR6)
2824	{
2825	if (!pVCpu->hmr0.s.fUsingHyperDR7)
2826	pCtx->dr[6] = pVmcbGuest->u64DR6;
2827	else
2828	CPUMSetHyperDR6(pVCpu, pVmcbGuest->u64DR6);
2829	}
2830
2831	if (fWhat & CPUMCTX_EXTRN_DR7)
2832	{
2833	if (!pVCpu->hmr0.s.fUsingHyperDR7)
2834	pCtx->dr[7] = pVmcbGuest->u64DR7;
2835	else
2836	Assert(pVmcbGuest->u64DR7 == CPUMGetHyperDR7(pVCpu));
2837	}
2838	}
2839
2840	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
2841	{
2842	if (fWhat & CPUMCTX_EXTRN_CR0)
2843	{
2844	/* We intercept changes to all CR0 bits except maybe TS & MP bits. */
2845	uint64_t const uCr0 = (pCtx->cr0 & ~(X86_CR0_TS \| X86_CR0_MP))
2846	\| (pVmcbGuest->u64CR0 & (X86_CR0_TS \| X86_CR0_MP));
2847	VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
2848	CPUMSetGuestCR0(pVCpu, uCr0);
2849	VMMRZCallRing3Enable(pVCpu);
2850	}
2851
2852	if (fWhat & CPUMCTX_EXTRN_CR2)
2853	pCtx->cr2 = pVmcbGuest->u64CR2;
2854
2855	if (fWhat & CPUMCTX_EXTRN_CR3)
2856	{
2857	if ( pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging
2858	&& pCtx->cr3 != pVmcbGuest->u64CR3)
2859	{
2860	CPUMSetGuestCR3(pVCpu, pVmcbGuest->u64CR3);
2861	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
2862	}
2863	}
2864
2865	/* Changes to CR4 are always intercepted. */
2866	}
2867
2868	/* Update fExtrn. */
2869	pCtx->fExtrn &= ~fWhat;
2870
2871	/* If everything has been imported, clear the HM keeper bit. */
2872	if (!(pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL))
2873	{
2874	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
2875	Assert(!pCtx->fExtrn);
2876	}
2877	}
2878	else
2879	Assert(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
2880
2881	ASMSetFlags(fEFlags);
2882
2883	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
2884
2885	/*
2886	* Honor any pending CR3 updates.
2887	*
2888	* Consider this scenario: #VMEXIT -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp
2889	* -> SVMR0CallRing3Callback() -> VMMRZCallRing3Disable() -> hmR0SvmImportGuestState()
2890	* -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp -> continue with #VMEXIT
2891	* handling -> hmR0SvmImportGuestState() and here we are.
2892	*
2893	* The reason for such complicated handling is because VM-exits that call into PGM expect
2894	* CR3 to be up-to-date and thus any CR3-saves -before- the VM-exit (longjmp) would've
2895	* postponed the CR3 update via the force-flag and cleared CR3 from fExtrn. Any SVM R0
2896	* VM-exit handler that requests CR3 to be saved will end up here and we call PGMUpdateCR3().
2897	*
2898	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again,
2899	* and does not process force-flag like regular exits to ring-3 either, we cover for it here.
2900	*/
2901	if ( VMMRZCallRing3IsEnabled(pVCpu)
2902	&& VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
2903	{
2904	AssertMsg(pCtx->cr3 == pVmcbGuest->u64CR3, ("cr3=%#RX64 vmcb_cr3=%#RX64\n", pCtx->cr3, pVmcbGuest->u64CR3));
2905	PGMUpdateCR3(pVCpu, pCtx->cr3);
2906	}
2907	}
2908
2909
2910	/**
2911	* Saves the guest (or nested-guest) state from the VMCB into the guest-CPU
2912	* context.
2913	*
2914	* Currently there is no residual state left in the CPU that is not updated in the
2915	* VMCB.
2916	*
2917	* @returns VBox status code.
2918	* @param pVCpu The cross context virtual CPU structure.
2919	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
2920	*/
2921	VMMR0DECL(int) SVMR0ImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
2922	{
2923	hmR0SvmImportGuestState(pVCpu, fWhat);
2924	return VINF_SUCCESS;
2925	}
2926
2927
2928	/**
2929	* Gets SVM \#VMEXIT auxiliary information.
2930	*
2931	* @returns VBox status code.
2932	* @param pVCpu The cross context virtual CPU structure.
2933	* @param pSvmExitAux Where to store the auxiliary info.
2934	*/
2935	VMMR0DECL(int) SVMR0GetExitAuxInfo(PVMCPUCC pVCpu, PSVMEXITAUX pSvmExitAux)
2936	{
2937	PCSVMTRANSIENT pSvmTransient = pVCpu->hmr0.s.svm.pSvmTransient;
2938	if (RT_LIKELY(pSvmTransient))
2939	{
2940	PCSVMVMCB pVmcb = pSvmTransient->pVmcb;
2941	if (RT_LIKELY(pVmcb))
2942	{
2943	pSvmExitAux->u64ExitCode = pVmcb->ctrl.u64ExitCode;
2944	pSvmExitAux->u64ExitInfo1 = pVmcb->ctrl.u64ExitInfo1;
2945	pSvmExitAux->u64ExitInfo2 = pVmcb->ctrl.u64ExitInfo2;
2946	pSvmExitAux->ExitIntInfo = pVmcb->ctrl.ExitIntInfo;
2947	return VINF_SUCCESS;
2948	}
2949	return VERR_SVM_IPE_5;
2950	}
2951	return VERR_NOT_AVAILABLE;
2952	}
2953
2954
2955	/**
2956	* Does the necessary state syncing before returning to ring-3 for any reason
2957	* (longjmp, preemption, voluntary exits to ring-3) from AMD-V.
2958	*
2959	* @param pVCpu The cross context virtual CPU structure.
2960	* @param fImportState Whether to import the guest state from the VMCB back
2961	* to the guest-CPU context.
2962	*
2963	* @remarks No-long-jmp zone!!!
2964	*/
2965	static void hmR0SvmLeave(PVMCPUCC pVCpu, bool fImportState)
2966	{
2967	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2968	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2969
2970	/*
2971	* !!! IMPORTANT !!!
2972	* If you modify code here, make sure to check whether SVMR0CallRing3Callback() needs to be updated too.
2973	*/
2974
2975	/* Save the guest state if necessary. */
2976	if (fImportState)
2977	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
2978
2979	/* Restore host FPU state if necessary and resync on next R0 reentry. */
2980	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
2981	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
2982
2983	/*
2984	* Restore host debug registers if necessary and resync on next R0 reentry.
2985	*/
2986	#ifdef VBOX_STRICT
2987	if (CPUMIsHyperDebugStateActive(pVCpu))
2988	{
2989	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb; /** @todo nested-guest. */
2990	Assert(pVmcb->ctrl.u16InterceptRdDRx == 0xffff);
2991	Assert(pVmcb->ctrl.u16InterceptWrDRx == 0xffff);
2992	}
2993	#endif
2994	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
2995	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
2996	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
2997
2998	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
2999	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
3000	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
3001	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
3002	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
3003	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitVmentry);
3004	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
3005
3006	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
3007	}
3008
3009
3010	/**
3011	* Leaves the AMD-V session.
3012	*
3013	* Only used while returning to ring-3 either due to longjump or exits to
3014	* ring-3.
3015	*
3016	* @returns VBox status code.
3017	* @param pVCpu The cross context virtual CPU structure.
3018	*/
3019	static int hmR0SvmLeaveSession(PVMCPUCC pVCpu)
3020	{
3021	HM_DISABLE_PREEMPT(pVCpu);
3022	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3023	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3024
3025	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
3026	and done this from the SVMR0ThreadCtxCallback(). */
3027	if (!pVCpu->hmr0.s.fLeaveDone)
3028	{
3029	hmR0SvmLeave(pVCpu, true /* fImportState */);
3030	pVCpu->hmr0.s.fLeaveDone = true;
3031	}
3032
3033	/*
3034	* !!! IMPORTANT !!!
3035	* If you modify code here, make sure to check whether SVMR0CallRing3Callback() needs to be updated too.
3036	*/
3037
3038	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
3039	/* Deregister hook now that we've left HM context before re-enabling preemption. */
3040	VMMR0ThreadCtxHookDisable(pVCpu);
3041
3042	/* Leave HM context. This takes care of local init (term). */
3043	int rc = HMR0LeaveCpu(pVCpu);
3044
3045	HM_RESTORE_PREEMPT();
3046	return rc;
3047	}
3048
3049
3050	/**
3051	* VMMRZCallRing3() callback wrapper which saves the guest state (or restores
3052	* any remaining host state) before we go back to ring-3 due to an assertion.
3053	*
3054	* @param pVCpu The cross context virtual CPU structure.
3055	*/
3056	VMMR0DECL(int) SVMR0AssertionCallback(PVMCPUCC pVCpu)
3057	{
3058	/*
3059	* !!! IMPORTANT !!!
3060	* If you modify code here, make sure to check whether hmR0SvmLeave() and hmR0SvmLeaveSession() needs
3061	* to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
3062	*/
3063	VMMR0AssertionRemoveNotification(pVCpu);
3064	VMMRZCallRing3Disable(pVCpu);
3065	HM_DISABLE_PREEMPT(pVCpu);
3066
3067	/* Import the entire guest state. */
3068	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3069
3070	/* Restore host FPU state if necessary and resync on next R0 reentry. */
3071	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
3072
3073	/* Restore host debug registers if necessary and resync on next R0 reentry. */
3074	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
3075
3076	/* Deregister the hook now that we've left HM context before re-enabling preemption. */
3077	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
3078	VMMR0ThreadCtxHookDisable(pVCpu);
3079
3080	/* Leave HM context. This takes care of local init (term). */
3081	HMR0LeaveCpu(pVCpu);
3082
3083	HM_RESTORE_PREEMPT();
3084	return VINF_SUCCESS;
3085	}
3086
3087
3088	/**
3089	* Take necessary actions before going back to ring-3.
3090	*
3091	* An action requires us to go back to ring-3. This function does the necessary
3092	* steps before we can safely return to ring-3. This is not the same as longjmps
3093	* to ring-3, this is voluntary.
3094	*
3095	* @returns Strict VBox status code.
3096	* @param pVCpu The cross context virtual CPU structure.
3097	* @param rcExit The reason for exiting to ring-3. Can be
3098	* VINF_VMM_UNKNOWN_RING3_CALL.
3099	*/
3100	static VBOXSTRICTRC hmR0SvmExitToRing3(PVMCPUCC pVCpu, VBOXSTRICTRC rcExit)
3101	{
3102	Assert(pVCpu);
3103	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
3104
3105	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
3106	VMMRZCallRing3Disable(pVCpu);
3107	Log4Func(("rcExit=%d LocalFF=%#RX64 GlobalFF=%#RX32\n", VBOXSTRICTRC_VAL(rcExit), (uint64_t)pVCpu->fLocalForcedActions,
3108	pVCpu->CTX_SUFF(pVM)->fGlobalForcedActions));
3109
3110	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
3111	if (pVCpu->hm.s.Event.fPending)
3112	{
3113	hmR0SvmPendingEventToTrpmTrap(pVCpu);
3114	Assert(!pVCpu->hm.s.Event.fPending);
3115	}
3116
3117	/* Sync. the necessary state for going back to ring-3. */
3118	hmR0SvmLeaveSession(pVCpu);
3119	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
3120
3121	/* Thread-context hooks are unregistered at this point!!! */
3122	/* Ring-3 callback notifications are unregistered at this point!!! */
3123
3124	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
3125	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
3126	\| CPUM_CHANGED_LDTR
3127	\| CPUM_CHANGED_GDTR
3128	\| CPUM_CHANGED_IDTR
3129	\| CPUM_CHANGED_TR
3130	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
3131	if ( pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging
3132	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
3133	{
3134	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
3135	}
3136
3137	/* Update the exit-to-ring 3 reason. */
3138	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
3139
3140	/* On our way back from ring-3, reload the guest-CPU state if it may change while in ring-3. */
3141	if ( rcExit != VINF_EM_RAW_INTERRUPT
3142	\|\| CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
3143	{
3144	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
3145	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
3146	}
3147
3148	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
3149	VMMRZCallRing3Enable(pVCpu);
3150
3151	/*
3152	* If we're emulating an instruction, we shouldn't have any TRPM traps pending
3153	* and if we're injecting an event we should have a TRPM trap pending.
3154	*/
3155	AssertReturnStmt(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu),
3156	pVCpu->hm.s.u32HMError = VBOXSTRICTRC_VAL(rcExit),
3157	VERR_SVM_IPE_5);
3158	AssertReturnStmt(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu),
3159	pVCpu->hm.s.u32HMError = VBOXSTRICTRC_VAL(rcExit),
3160	VERR_SVM_IPE_4);
3161
3162	return rcExit;
3163	}
3164
3165
3166	/**
3167	* Updates the use of TSC offsetting mode for the CPU and adjusts the necessary
3168	* intercepts.
3169	*
3170	* @param pVCpu The cross context virtual CPU structure.
3171	* @param pVmcb Pointer to the VM control block.
3172	*
3173	* @remarks No-long-jump zone!!!
3174	*/
3175	static void hmR0SvmUpdateTscOffsetting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3176	{
3177	/*
3178	* Avoid intercepting RDTSC/RDTSCP if we determined the host TSC (++) is stable
3179	* and in case of a nested-guest, if the nested-VMCB specifies it is not intercepting
3180	* RDTSC/RDTSCP as well.
3181	*/
3182	bool fParavirtTsc;
3183	uint64_t uTscOffset;
3184	bool const fCanUseRealTsc = TMCpuTickCanUseRealTSC(pVCpu->CTX_SUFF(pVM), pVCpu, &uTscOffset, &fParavirtTsc);
3185
3186	bool fIntercept;
3187	if (fCanUseRealTsc)
3188	fIntercept = hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP);
3189	else
3190	{
3191	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP);
3192	fIntercept = true;
3193	}
3194
3195	if (!fIntercept)
3196	{
3197	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3198	/* Apply the nested-guest VMCB's TSC offset over the guest TSC offset. */
3199	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
3200	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
3201	#endif
3202
3203	/* Update the TSC offset in the VMCB and the relevant clean bits. */
3204	pVmcb->ctrl.u64TSCOffset = uTscOffset;
3205	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
3206	}
3207
3208	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
3209	information before every VM-entry, hence we have nothing to do here at the moment. */
3210	if (fParavirtTsc)
3211	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
3212	}
3213
3214
3215	/**
3216	* Sets an event as a pending event to be injected into the guest.
3217	*
3218	* @param pVCpu The cross context virtual CPU structure.
3219	* @param pEvent Pointer to the SVM event.
3220	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
3221	* page-fault.
3222	*
3223	* @remarks Statistics counter assumes this is a guest event being reflected to
3224	* the guest i.e. 'StatInjectPendingReflect' is incremented always.
3225	*/
3226	DECLINLINE(void) hmR0SvmSetPendingEvent(PVMCPUCC pVCpu, PSVMEVENT pEvent, RTGCUINTPTR GCPtrFaultAddress)
3227	{
3228	Assert(!pVCpu->hm.s.Event.fPending);
3229	Assert(pEvent->n.u1Valid);
3230
3231	pVCpu->hm.s.Event.u64IntInfo = pEvent->u;
3232	pVCpu->hm.s.Event.fPending = true;
3233	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
3234
3235	Log4Func(("u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u, pEvent->n.u8Vector,
3236	(uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3237	}
3238
3239
3240	/**
3241	* Sets an divide error (\#DE) exception as pending-for-injection into the VM.
3242	*
3243	* @param pVCpu The cross context virtual CPU structure.
3244	*/
3245	DECLINLINE(void) hmR0SvmSetPendingXcptDE(PVMCPUCC pVCpu)
3246	{
3247	SVMEVENT Event;
3248	Event.u = 0;
3249	Event.n.u1Valid = 1;
3250	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3251	Event.n.u8Vector = X86_XCPT_DE;
3252	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3253	}
3254
3255
3256	/**
3257	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
3258	*
3259	* @param pVCpu The cross context virtual CPU structure.
3260	*/
3261	DECLINLINE(void) hmR0SvmSetPendingXcptUD(PVMCPUCC pVCpu)
3262	{
3263	SVMEVENT Event;
3264	Event.u = 0;
3265	Event.n.u1Valid = 1;
3266	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3267	Event.n.u8Vector = X86_XCPT_UD;
3268	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3269	}
3270
3271
3272	/**
3273	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
3274	*
3275	* @param pVCpu The cross context virtual CPU structure.
3276	*/
3277	DECLINLINE(void) hmR0SvmSetPendingXcptDB(PVMCPUCC pVCpu)
3278	{
3279	SVMEVENT Event;
3280	Event.u = 0;
3281	Event.n.u1Valid = 1;
3282	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3283	Event.n.u8Vector = X86_XCPT_DB;
3284	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3285	}
3286
3287
3288	/**
3289	* Sets a page fault (\#PF) exception as pending-for-injection into the VM.
3290	*
3291	* @param pVCpu The cross context virtual CPU structure.
3292	* @param u32ErrCode The error-code for the page-fault.
3293	* @param uFaultAddress The page fault address (CR2).
3294	*
3295	* @remarks This updates the guest CR2 with @a uFaultAddress!
3296	*/
3297	DECLINLINE(void) hmR0SvmSetPendingXcptPF(PVMCPUCC pVCpu, uint32_t u32ErrCode, RTGCUINTPTR uFaultAddress)
3298	{
3299	SVMEVENT Event;
3300	Event.u = 0;
3301	Event.n.u1Valid = 1;
3302	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3303	Event.n.u8Vector = X86_XCPT_PF;
3304	Event.n.u1ErrorCodeValid = 1;
3305	Event.n.u32ErrorCode = u32ErrCode;
3306
3307	/* Update CR2 of the guest. */
3308	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR2);
3309	if (pVCpu->cpum.GstCtx.cr2 != uFaultAddress)
3310	{
3311	pVCpu->cpum.GstCtx.cr2 = uFaultAddress;
3312	/* The VMCB clean bit for CR2 will be updated while re-loading the guest state. */
3313	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR2);
3314	}
3315
3316	hmR0SvmSetPendingEvent(pVCpu, &Event, uFaultAddress);
3317	}
3318
3319
3320	/**
3321	* Sets a math-fault (\#MF) exception as pending-for-injection into the VM.
3322	*
3323	* @param pVCpu The cross context virtual CPU structure.
3324	*/
3325	DECLINLINE(void) hmR0SvmSetPendingXcptMF(PVMCPUCC pVCpu)
3326	{
3327	SVMEVENT Event;
3328	Event.u = 0;
3329	Event.n.u1Valid = 1;
3330	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3331	Event.n.u8Vector = X86_XCPT_MF;
3332	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3333	}
3334
3335
3336	/**
3337	* Sets a double fault (\#DF) exception as pending-for-injection into the VM.
3338	*
3339	* @param pVCpu The cross context virtual CPU structure.
3340	*/
3341	DECLINLINE(void) hmR0SvmSetPendingXcptDF(PVMCPUCC pVCpu)
3342	{
3343	SVMEVENT Event;
3344	Event.u = 0;
3345	Event.n.u1Valid = 1;
3346	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3347	Event.n.u8Vector = X86_XCPT_DF;
3348	Event.n.u1ErrorCodeValid = 1;
3349	Event.n.u32ErrorCode = 0;
3350	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3351	}
3352
3353
3354	/**
3355	* Injects an event into the guest upon VMRUN by updating the relevant field
3356	* in the VMCB.
3357	*
3358	* @param pVCpu The cross context virtual CPU structure.
3359	* @param pVmcb Pointer to the guest VM control block.
3360	* @param pEvent Pointer to the event.
3361	*
3362	* @remarks No-long-jump zone!!!
3363	* @remarks Requires CR0!
3364	*/
3365	DECLINLINE(void) hmR0SvmInjectEventVmcb(PVMCPUCC pVCpu, PSVMVMCB pVmcb, PSVMEVENT pEvent)
3366	{
3367	Assert(!pVmcb->ctrl.EventInject.n.u1Valid);
3368	pVmcb->ctrl.EventInject.u = pEvent->u;
3369	if ( pVmcb->ctrl.EventInject.n.u3Type == SVM_EVENT_EXCEPTION
3370	\|\| pVmcb->ctrl.EventInject.n.u3Type == SVM_EVENT_NMI)
3371	{
3372	Assert(pEvent->n.u8Vector <= X86_XCPT_LAST);
3373	STAM_COUNTER_INC(&pVCpu->hm.s.aStatInjectedXcpts[pEvent->n.u8Vector]);
3374	}
3375	else
3376	STAM_COUNTER_INC(&pVCpu->hm.s.aStatInjectedIrqs[pEvent->n.u8Vector & MASK_INJECT_IRQ_STAT]);
3377	RT_NOREF(pVCpu);
3378
3379	Log4Func(("u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u, pEvent->n.u8Vector,
3380	(uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3381	}
3382
3383
3384
3385	/**
3386	* Converts any TRPM trap into a pending HM event. This is typically used when
3387	* entering from ring-3 (not longjmp returns).
3388	*
3389	* @param pVCpu The cross context virtual CPU structure.
3390	*/
3391	static void hmR0SvmTrpmTrapToPendingEvent(PVMCPUCC pVCpu)
3392	{
3393	Assert(TRPMHasTrap(pVCpu));
3394	Assert(!pVCpu->hm.s.Event.fPending);
3395
3396	uint8_t uVector;
3397	TRPMEVENT enmTrpmEvent;
3398	uint32_t uErrCode;
3399	RTGCUINTPTR GCPtrFaultAddress;
3400	uint8_t cbInstr;
3401
3402	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr, NULL /* pfIcebp */);
3403	AssertRC(rc);
3404
3405	SVMEVENT Event;
3406	Event.u = 0;
3407	Event.n.u1Valid = 1;
3408	Event.n.u8Vector = uVector;
3409
3410	/* Refer AMD spec. 15.20 "Event Injection" for the format. */
3411	if (enmTrpmEvent == TRPM_TRAP)
3412	{
3413	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3414	switch (uVector)
3415	{
3416	case X86_XCPT_NMI:
3417	{
3418	Event.n.u3Type = SVM_EVENT_NMI;
3419	break;
3420	}
3421
3422	case X86_XCPT_BP:
3423	case X86_XCPT_OF:
3424	AssertMsgFailed(("Invalid TRPM vector %d for event type %d\n", uVector, enmTrpmEvent));
3425	RT_FALL_THRU();
3426
3427	case X86_XCPT_PF:
3428	case X86_XCPT_DF:
3429	case X86_XCPT_TS:
3430	case X86_XCPT_NP:
3431	case X86_XCPT_SS:
3432	case X86_XCPT_GP:
3433	case X86_XCPT_AC:
3434	{
3435	Event.n.u1ErrorCodeValid = 1;
3436	Event.n.u32ErrorCode = uErrCode;
3437	break;
3438	}
3439	}
3440	}
3441	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
3442	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3443	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
3444	Event.n.u3Type = SVM_EVENT_SOFTWARE_INT;
3445	else
3446	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
3447
3448	rc = TRPMResetTrap(pVCpu);
3449	AssertRC(rc);
3450
3451	Log4(("TRPM->HM event: u=%#RX64 u8Vector=%#x uErrorCodeValid=%RTbool uErrorCode=%#RX32\n", Event.u, Event.n.u8Vector,
3452	!!Event.n.u1ErrorCodeValid, Event.n.u32ErrorCode));
3453
3454	hmR0SvmSetPendingEvent(pVCpu, &Event, GCPtrFaultAddress);
3455	}
3456
3457
3458	/**
3459	* Converts any pending SVM event into a TRPM trap. Typically used when leaving
3460	* AMD-V to execute any instruction.
3461	*
3462	* @param pVCpu The cross context virtual CPU structure.
3463	*/
3464	static void hmR0SvmPendingEventToTrpmTrap(PVMCPUCC pVCpu)
3465	{
3466	Assert(pVCpu->hm.s.Event.fPending);
3467	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
3468
3469	SVMEVENT Event;
3470	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3471
3472	uint8_t uVector = Event.n.u8Vector;
3473	TRPMEVENT enmTrapType = HMSvmEventToTrpmEventType(&Event, uVector);
3474
3475	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, Event.n.u3Type));
3476
3477	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
3478	AssertRC(rc);
3479
3480	if (Event.n.u1ErrorCodeValid)
3481	TRPMSetErrorCode(pVCpu, Event.n.u32ErrorCode);
3482
3483	if ( enmTrapType == TRPM_TRAP
3484	&& uVector == X86_XCPT_PF)
3485	{
3486	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
3487	Assert(pVCpu->hm.s.Event.GCPtrFaultAddress == CPUMGetGuestCR2(pVCpu));
3488	}
3489	else if (enmTrapType == TRPM_SOFTWARE_INT)
3490	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
3491	pVCpu->hm.s.Event.fPending = false;
3492	}
3493
3494
3495	/**
3496	* Checks if the guest (or nested-guest) has an interrupt shadow active right
3497	* now.
3498	*
3499	* @returns @c true if the interrupt shadow is active, @c false otherwise.
3500	* @param pVCpu The cross context virtual CPU structure.
3501	*
3502	* @remarks No-long-jump zone!!!
3503	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3504	*/
3505	static bool hmR0SvmIsIntrShadowActive(PVMCPUCC pVCpu)
3506	{
3507	/*
3508	* Instructions like STI and MOV SS inhibit interrupts till the next instruction
3509	* completes. Check if we should inhibit interrupts or clear any existing
3510	* interrupt inhibition.
3511	*/
3512	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3513	{
3514	if (pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
3515	{
3516	/*
3517	* We can clear the inhibit force flag as even if we go back to the recompiler
3518	* without executing guest code in AMD-V, the flag's condition to be cleared is
3519	* met and thus the cleared state is correct.
3520	*/
3521	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3522	return false;
3523	}
3524	return true;
3525	}
3526	return false;
3527	}
3528
3529
3530	/**
3531	* Sets the virtual interrupt intercept control in the VMCB.
3532	*
3533	* @param pVCpu The cross context virtual CPU structure.
3534	* @param pVmcb Pointer to the VM control block.
3535	*/
3536	static void hmR0SvmSetIntWindowExiting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3537	{
3538	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx); NOREF(pVCpu);
3539
3540	/*
3541	* When AVIC isn't supported, set up an interrupt window to cause a #VMEXIT when the guest
3542	* is ready to accept interrupts. At #VMEXIT, we then get the interrupt from the APIC
3543	* (updating ISR at the right time) and inject the interrupt.
3544	*
3545	* With AVIC is supported, we could make use of the asynchronously delivery without
3546	* #VMEXIT and we would be passing the AVIC page to SVM.
3547	*
3548	* In AMD-V, an interrupt window is achieved using a combination of V_IRQ (an interrupt
3549	* is pending), V_IGN_TPR (ignore TPR priorities) and the VINTR intercept all being set.
3550	*/
3551	Assert(pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR);
3552	pVmcb->ctrl.IntCtrl.n.u1VIrqPending = 1;
3553	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
3554	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_VINTR);
3555	Log4(("Set VINTR intercept\n"));
3556	}
3557
3558
3559	/**
3560	* Clears the virtual interrupt intercept control in the VMCB as
3561	* we are figured the guest is unable process any interrupts
3562	* at this point of time.
3563	*
3564	* @param pVCpu The cross context virtual CPU structure.
3565	* @param pVmcb Pointer to the VM control block.
3566	*/
3567	static void hmR0SvmClearIntWindowExiting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3568	{
3569	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx); NOREF(pVCpu);
3570
3571	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
3572	if ( pVmcbCtrl->IntCtrl.n.u1VIrqPending
3573	\|\| (pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_VINTR))
3574	{
3575	pVmcbCtrl->IntCtrl.n.u1VIrqPending = 0;
3576	pVmcbCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
3577	hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_VINTR);
3578	Log4(("Cleared VINTR intercept\n"));
3579	}
3580	}
3581
3582
3583	/**
3584	* Evaluates the event to be delivered to the guest and sets it as the pending
3585	* event.
3586	*
3587	* @returns Strict VBox status code.
3588	* @param pVCpu The cross context virtual CPU structure.
3589	* @param pSvmTransient Pointer to the SVM transient structure.
3590	*/
3591	static VBOXSTRICTRC hmR0SvmEvaluatePendingEvent(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient)
3592	{
3593	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3594	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_HWVIRT
3595	\| CPUMCTX_EXTRN_RFLAGS
3596	\| CPUMCTX_EXTRN_INHIBIT_INT
3597	\| CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ);
3598
3599	Assert(!pVCpu->hm.s.Event.fPending);
3600	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
3601	Assert(pVmcb);
3602
3603	bool const fGif = CPUMGetGuestGif(pCtx);
3604	bool const fIntShadow = hmR0SvmIsIntrShadowActive(pVCpu);
3605	bool const fBlockNmi = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
3606
3607	Log4Func(("fGif=%RTbool fBlockNmi=%RTbool fIntShadow=%RTbool fIntPending=%RTbool fNmiPending=%RTbool\n",
3608	fGif, fBlockNmi, fIntShadow, VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC),
3609	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)));
3610
3611	/** @todo SMI. SMIs take priority over NMIs. */
3612
3613	/*
3614	* Check if the guest or nested-guest can receive NMIs.
3615	* Nested NMIs are not allowed, see AMD spec. 8.1.4 "Masking External Interrupts".
3616	* NMIs take priority over maskable interrupts, see AMD spec. 8.5 "Priorities".
3617	*/
3618	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)
3619	&& !fBlockNmi)
3620	{
3621	if ( fGif
3622	&& !fIntShadow)
3623	{
3624	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3625	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_NMI))
3626	{
3627	Log4(("Intercepting NMI -> #VMEXIT\n"));
3628	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3629	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_NMI, 0, 0);
3630	}
3631	#endif
3632	Log4(("Setting NMI pending for injection\n"));
3633	SVMEVENT Event;
3634	Event.u = 0;
3635	Event.n.u1Valid = 1;
3636	Event.n.u8Vector = X86_XCPT_NMI;
3637	Event.n.u3Type = SVM_EVENT_NMI;
3638	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3639	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
3640	}
3641	else if (!fGif)
3642	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3643	else if (!pSvmTransient->fIsNestedGuest)
3644	hmR0SvmSetIntWindowExiting(pVCpu, pVmcb);
3645	/* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCB controls. */
3646	}
3647	/*
3648	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt()
3649	* returns a valid interrupt we -must- deliver the interrupt. We can no longer re-request
3650	* it from the APIC device.
3651	*
3652	* For nested-guests, physical interrupts always take priority over virtual interrupts.
3653	* We don't need to inject nested-guest virtual interrupts here, we can let the hardware
3654	* do that work when we execute nested-guest code esp. since all the required information
3655	* is in the VMCB, unlike physical interrupts where we need to fetch the interrupt from
3656	* the virtual interrupt controller.
3657	*
3658	* See AMD spec. 15.21.4 "Injecting Virtual (INTR) Interrupts".
3659	*/
3660	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
3661	&& !pVCpu->hm.s.fSingleInstruction)
3662	{
3663	bool const fBlockInt = !pSvmTransient->fIsNestedGuest ? !(pCtx->eflags.u32 & X86_EFL_IF)
3664	: CPUMIsGuestSvmPhysIntrEnabled(pVCpu, pCtx);
3665	if ( fGif
3666	&& !fBlockInt
3667	&& !fIntShadow)
3668	{
3669	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3670	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INTR))
3671	{
3672	Log4(("Intercepting INTR -> #VMEXIT\n"));
3673	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3674	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_INTR, 0, 0);
3675	}
3676	#endif
3677	uint8_t u8Interrupt;
3678	int rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
3679	if (RT_SUCCESS(rc))
3680	{
3681	Log4(("Setting external interrupt %#x pending for injection\n", u8Interrupt));
3682	SVMEVENT Event;
3683	Event.u = 0;
3684	Event.n.u1Valid = 1;
3685	Event.n.u8Vector = u8Interrupt;
3686	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3687	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3688	}
3689	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
3690	{
3691	/*
3692	* AMD-V has no TPR thresholding feature. TPR and the force-flag will be
3693	* updated eventually when the TPR is written by the guest.
3694	*/
3695	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
3696	}
3697	else
3698	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
3699	}
3700	else if (!fGif)
3701	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3702	else if (!pSvmTransient->fIsNestedGuest)
3703	hmR0SvmSetIntWindowExiting(pVCpu, pVmcb);
3704	/* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCB controls. */
3705	}
3706
3707	return VINF_SUCCESS;
3708	}
3709
3710
3711	/**
3712	* Injects any pending events into the guest (or nested-guest).
3713	*
3714	* @param pVCpu The cross context virtual CPU structure.
3715	* @param pVmcb Pointer to the VM control block.
3716	*
3717	* @remarks Must only be called when we are guaranteed to enter
3718	* hardware-assisted SVM execution and not return to ring-3
3719	* prematurely.
3720	*/
3721	static void hmR0SvmInjectPendingEvent(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3722	{
3723	Assert(!TRPMHasTrap(pVCpu));
3724	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3725
3726	bool const fIntShadow = hmR0SvmIsIntrShadowActive(pVCpu);
3727	#ifdef VBOX_STRICT
3728	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3729	bool const fGif = CPUMGetGuestGif(pCtx);
3730	bool fAllowInt = fGif;
3731	if (fGif)
3732	{
3733	/*
3734	* For nested-guests we have no way to determine if we're injecting a physical or
3735	* virtual interrupt at this point. Hence the partial verification below.
3736	*/
3737	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
3738	fAllowInt = CPUMIsGuestSvmPhysIntrEnabled(pVCpu, pCtx) \|\| CPUMIsGuestSvmVirtIntrEnabled(pVCpu, pCtx);
3739	else
3740	fAllowInt = RT_BOOL(pCtx->eflags.u32 & X86_EFL_IF);
3741	}
3742	#endif
3743
3744	if (pVCpu->hm.s.Event.fPending)
3745	{
3746	SVMEVENT Event;
3747	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3748	Assert(Event.n.u1Valid);
3749
3750	/*
3751	* Validate event injection pre-conditions.
3752	*/
3753	if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3754	{
3755	Assert(fAllowInt);
3756	Assert(!fIntShadow);
3757	}
3758	else if (Event.n.u3Type == SVM_EVENT_NMI)
3759	{
3760	Assert(fGif);
3761	Assert(!fIntShadow);
3762	}
3763
3764	/*
3765	* Before injecting an NMI we must set VMCPU_FF_BLOCK_NMIS to prevent nested NMIs. We
3766	* do this only when we are surely going to inject the NMI as otherwise if we return
3767	* to ring-3 prematurely we could leave NMIs blocked indefinitely upon re-entry into
3768	* SVM R0.
3769	*
3770	* With VT-x, this is handled by the Guest interruptibility information VMCS field
3771	* which will set the VMCS field after actually delivering the NMI which we read on
3772	* VM-exit to determine the state.
3773	*/
3774	if ( Event.n.u3Type == SVM_EVENT_NMI
3775	&& Event.n.u8Vector == X86_XCPT_NMI
3776	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
3777	{
3778	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
3779	}
3780
3781	/*
3782	* Inject it (update VMCB for injection by the hardware).
3783	*/
3784	Log4(("Injecting pending HM event\n"));
3785	hmR0SvmInjectEventVmcb(pVCpu, pVmcb, &Event);
3786	pVCpu->hm.s.Event.fPending = false;
3787
3788	if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3789	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
3790	else
3791	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
3792	}
3793	else
3794	Assert(pVmcb->ctrl.EventInject.n.u1Valid == 0);
3795
3796	/*
3797	* We could have injected an NMI through IEM and continue guest execution using
3798	* hardware-assisted SVM. In which case, we would not have any events pending (above)
3799	* but we still need to intercept IRET in order to eventually clear NMI inhibition.
3800	*/
3801	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
3802	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_IRET);
3803
3804	/*
3805	* Update the guest interrupt shadow in the guest (or nested-guest) VMCB.
3806	*
3807	* For nested-guests: We need to update it too for the scenario where IEM executes
3808	* the nested-guest but execution later continues here with an interrupt shadow active.
3809	*/
3810	pVmcb->ctrl.IntShadow.n.u1IntShadow = fIntShadow;
3811	}
3812
3813
3814	/**
3815	* Reports world-switch error and dumps some useful debug info.
3816	*
3817	* @param pVCpu The cross context virtual CPU structure.
3818	* @param rcVMRun The return code from VMRUN (or
3819	* VERR_SVM_INVALID_GUEST_STATE for invalid
3820	* guest-state).
3821	*/
3822	static void hmR0SvmReportWorldSwitchError(PVMCPUCC pVCpu, int rcVMRun)
3823	{
3824	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
3825	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
3826	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3827
3828	if (rcVMRun == VERR_SVM_INVALID_GUEST_STATE)
3829	{
3830	#ifdef VBOX_STRICT
3831	hmR0DumpRegs(pVCpu, HM_DUMP_REG_FLAGS_ALL);
3832	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
3833	Log4(("ctrl.u32VmcbCleanBits %#RX32\n", pVmcb->ctrl.u32VmcbCleanBits));
3834	Log4(("ctrl.u16InterceptRdCRx %#x\n", pVmcb->ctrl.u16InterceptRdCRx));
3835	Log4(("ctrl.u16InterceptWrCRx %#x\n", pVmcb->ctrl.u16InterceptWrCRx));
3836	Log4(("ctrl.u16InterceptRdDRx %#x\n", pVmcb->ctrl.u16InterceptRdDRx));
3837	Log4(("ctrl.u16InterceptWrDRx %#x\n", pVmcb->ctrl.u16InterceptWrDRx));
3838	Log4(("ctrl.u32InterceptXcpt %#x\n", pVmcb->ctrl.u32InterceptXcpt));
3839	Log4(("ctrl.u64InterceptCtrl %#RX64\n", pVmcb->ctrl.u64InterceptCtrl));
3840	Log4(("ctrl.u64IOPMPhysAddr %#RX64\n", pVmcb->ctrl.u64IOPMPhysAddr));
3841	Log4(("ctrl.u64MSRPMPhysAddr %#RX64\n", pVmcb->ctrl.u64MSRPMPhysAddr));
3842	Log4(("ctrl.u64TSCOffset %#RX64\n", pVmcb->ctrl.u64TSCOffset));
3843
3844	Log4(("ctrl.TLBCtrl.u32ASID %#x\n", pVmcb->ctrl.TLBCtrl.n.u32ASID));
3845	Log4(("ctrl.TLBCtrl.u8TLBFlush %#x\n", pVmcb->ctrl.TLBCtrl.n.u8TLBFlush));
3846	Log4(("ctrl.TLBCtrl.u24Reserved %#x\n", pVmcb->ctrl.TLBCtrl.n.u24Reserved));
3847
3848	Log4(("ctrl.IntCtrl.u8VTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u8VTPR));
3849	Log4(("ctrl.IntCtrl.u1VIrqPending %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIrqPending));
3850	Log4(("ctrl.IntCtrl.u1VGif %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGif));
3851	Log4(("ctrl.IntCtrl.u6Reserved0 %#x\n", pVmcb->ctrl.IntCtrl.n.u6Reserved));
3852	Log4(("ctrl.IntCtrl.u4VIntrPrio %#x\n", pVmcb->ctrl.IntCtrl.n.u4VIntrPrio));
3853	Log4(("ctrl.IntCtrl.u1IgnoreTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR));
3854	Log4(("ctrl.IntCtrl.u3Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u3Reserved));
3855	Log4(("ctrl.IntCtrl.u1VIntrMasking %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIntrMasking));
3856	Log4(("ctrl.IntCtrl.u1VGifEnable %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGifEnable));
3857	Log4(("ctrl.IntCtrl.u5Reserved1 %#x\n", pVmcb->ctrl.IntCtrl.n.u5Reserved));
3858	Log4(("ctrl.IntCtrl.u8VIntrVector %#x\n", pVmcb->ctrl.IntCtrl.n.u8VIntrVector));
3859	Log4(("ctrl.IntCtrl.u24Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u24Reserved));
3860
3861	Log4(("ctrl.IntShadow.u1IntShadow %#x\n", pVmcb->ctrl.IntShadow.n.u1IntShadow));
3862	Log4(("ctrl.IntShadow.u1GuestIntMask %#x\n", pVmcb->ctrl.IntShadow.n.u1GuestIntMask));
3863	Log4(("ctrl.u64ExitCode %#RX64\n", pVmcb->ctrl.u64ExitCode));
3864	Log4(("ctrl.u64ExitInfo1 %#RX64\n", pVmcb->ctrl.u64ExitInfo1));
3865	Log4(("ctrl.u64ExitInfo2 %#RX64\n", pVmcb->ctrl.u64ExitInfo2));
3866	Log4(("ctrl.ExitIntInfo.u8Vector %#x\n", pVmcb->ctrl.ExitIntInfo.n.u8Vector));
3867	Log4(("ctrl.ExitIntInfo.u3Type %#x\n", pVmcb->ctrl.ExitIntInfo.n.u3Type));
3868	Log4(("ctrl.ExitIntInfo.u1ErrorCodeValid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid));
3869	Log4(("ctrl.ExitIntInfo.u19Reserved %#x\n", pVmcb->ctrl.ExitIntInfo.n.u19Reserved));
3870	Log4(("ctrl.ExitIntInfo.u1Valid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1Valid));
3871	Log4(("ctrl.ExitIntInfo.u32ErrorCode %#x\n", pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode));
3872	Log4(("ctrl.NestedPagingCtrl.u1NestedPaging %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1NestedPaging));
3873	Log4(("ctrl.NestedPagingCtrl.u1Sev %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1Sev));
3874	Log4(("ctrl.NestedPagingCtrl.u1SevEs %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1SevEs));
3875	Log4(("ctrl.EventInject.u8Vector %#x\n", pVmcb->ctrl.EventInject.n.u8Vector));
3876	Log4(("ctrl.EventInject.u3Type %#x\n", pVmcb->ctrl.EventInject.n.u3Type));
3877	Log4(("ctrl.EventInject.u1ErrorCodeValid %#x\n", pVmcb->ctrl.EventInject.n.u1ErrorCodeValid));
3878	Log4(("ctrl.EventInject.u19Reserved %#x\n", pVmcb->ctrl.EventInject.n.u19Reserved));
3879	Log4(("ctrl.EventInject.u1Valid %#x\n", pVmcb->ctrl.EventInject.n.u1Valid));
3880	Log4(("ctrl.EventInject.u32ErrorCode %#x\n", pVmcb->ctrl.EventInject.n.u32ErrorCode));
3881
3882	Log4(("ctrl.u64NestedPagingCR3 %#RX64\n", pVmcb->ctrl.u64NestedPagingCR3));
3883
3884	Log4(("ctrl.LbrVirt.u1LbrVirt %#x\n", pVmcb->ctrl.LbrVirt.n.u1LbrVirt));
3885	Log4(("ctrl.LbrVirt.u1VirtVmsaveVmload %#x\n", pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload));
3886
3887	Log4(("guest.CS.u16Sel %RTsel\n", pVmcb->guest.CS.u16Sel));
3888	Log4(("guest.CS.u16Attr %#x\n", pVmcb->guest.CS.u16Attr));
3889	Log4(("guest.CS.u32Limit %#RX32\n", pVmcb->guest.CS.u32Limit));
3890	Log4(("guest.CS.u64Base %#RX64\n", pVmcb->guest.CS.u64Base));
3891	Log4(("guest.DS.u16Sel %#RTsel\n", pVmcb->guest.DS.u16Sel));
3892	Log4(("guest.DS.u16Attr %#x\n", pVmcb->guest.DS.u16Attr));
3893	Log4(("guest.DS.u32Limit %#RX32\n", pVmcb->guest.DS.u32Limit));
3894	Log4(("guest.DS.u64Base %#RX64\n", pVmcb->guest.DS.u64Base));
3895	Log4(("guest.ES.u16Sel %RTsel\n", pVmcb->guest.ES.u16Sel));
3896	Log4(("guest.ES.u16Attr %#x\n", pVmcb->guest.ES.u16Attr));
3897	Log4(("guest.ES.u32Limit %#RX32\n", pVmcb->guest.ES.u32Limit));
3898	Log4(("guest.ES.u64Base %#RX64\n", pVmcb->guest.ES.u64Base));
3899	Log4(("guest.FS.u16Sel %RTsel\n", pVmcb->guest.FS.u16Sel));
3900	Log4(("guest.FS.u16Attr %#x\n", pVmcb->guest.FS.u16Attr));
3901	Log4(("guest.FS.u32Limit %#RX32\n", pVmcb->guest.FS.u32Limit));
3902	Log4(("guest.FS.u64Base %#RX64\n", pVmcb->guest.FS.u64Base));
3903	Log4(("guest.GS.u16Sel %RTsel\n", pVmcb->guest.GS.u16Sel));
3904	Log4(("guest.GS.u16Attr %#x\n", pVmcb->guest.GS.u16Attr));
3905	Log4(("guest.GS.u32Limit %#RX32\n", pVmcb->guest.GS.u32Limit));
3906	Log4(("guest.GS.u64Base %#RX64\n", pVmcb->guest.GS.u64Base));
3907
3908	Log4(("guest.GDTR.u32Limit %#RX32\n", pVmcb->guest.GDTR.u32Limit));
3909	Log4(("guest.GDTR.u64Base %#RX64\n", pVmcb->guest.GDTR.u64Base));
3910
3911	Log4(("guest.LDTR.u16Sel %RTsel\n", pVmcb->guest.LDTR.u16Sel));
3912	Log4(("guest.LDTR.u16Attr %#x\n", pVmcb->guest.LDTR.u16Attr));
3913	Log4(("guest.LDTR.u32Limit %#RX32\n", pVmcb->guest.LDTR.u32Limit));
3914	Log4(("guest.LDTR.u64Base %#RX64\n", pVmcb->guest.LDTR.u64Base));
3915
3916	Log4(("guest.IDTR.u32Limit %#RX32\n", pVmcb->guest.IDTR.u32Limit));
3917	Log4(("guest.IDTR.u64Base %#RX64\n", pVmcb->guest.IDTR.u64Base));
3918
3919	Log4(("guest.TR.u16Sel %RTsel\n", pVmcb->guest.TR.u16Sel));
3920	Log4(("guest.TR.u16Attr %#x\n", pVmcb->guest.TR.u16Attr));
3921	Log4(("guest.TR.u32Limit %#RX32\n", pVmcb->guest.TR.u32Limit));
3922	Log4(("guest.TR.u64Base %#RX64\n", pVmcb->guest.TR.u64Base));
3923
3924	Log4(("guest.u8CPL %#x\n", pVmcb->guest.u8CPL));
3925	Log4(("guest.u64CR0 %#RX64\n", pVmcb->guest.u64CR0));
3926	Log4(("guest.u64CR2 %#RX64\n", pVmcb->guest.u64CR2));
3927	Log4(("guest.u64CR3 %#RX64\n", pVmcb->guest.u64CR3));
3928	Log4(("guest.u64CR4 %#RX64\n", pVmcb->guest.u64CR4));
3929	Log4(("guest.u64DR6 %#RX64\n", pVmcb->guest.u64DR6));
3930	Log4(("guest.u64DR7 %#RX64\n", pVmcb->guest.u64DR7));
3931
3932	Log4(("guest.u64RIP %#RX64\n", pVmcb->guest.u64RIP));
3933	Log4(("guest.u64RSP %#RX64\n", pVmcb->guest.u64RSP));
3934	Log4(("guest.u64RAX %#RX64\n", pVmcb->guest.u64RAX));
3935	Log4(("guest.u64RFlags %#RX64\n", pVmcb->guest.u64RFlags));
3936
3937	Log4(("guest.u64SysEnterCS %#RX64\n", pVmcb->guest.u64SysEnterCS));
3938	Log4(("guest.u64SysEnterEIP %#RX64\n", pVmcb->guest.u64SysEnterEIP));
3939	Log4(("guest.u64SysEnterESP %#RX64\n", pVmcb->guest.u64SysEnterESP));
3940
3941	Log4(("guest.u64EFER %#RX64\n", pVmcb->guest.u64EFER));
3942	Log4(("guest.u64STAR %#RX64\n", pVmcb->guest.u64STAR));
3943	Log4(("guest.u64LSTAR %#RX64\n", pVmcb->guest.u64LSTAR));
3944	Log4(("guest.u64CSTAR %#RX64\n", pVmcb->guest.u64CSTAR));
3945	Log4(("guest.u64SFMASK %#RX64\n", pVmcb->guest.u64SFMASK));
3946	Log4(("guest.u64KernelGSBase %#RX64\n", pVmcb->guest.u64KernelGSBase));
3947	Log4(("guest.u64PAT %#RX64\n", pVmcb->guest.u64PAT));
3948	Log4(("guest.u64DBGCTL %#RX64\n", pVmcb->guest.u64DBGCTL));
3949	Log4(("guest.u64BR_FROM %#RX64\n", pVmcb->guest.u64BR_FROM));
3950	Log4(("guest.u64BR_TO %#RX64\n", pVmcb->guest.u64BR_TO));
3951	Log4(("guest.u64LASTEXCPFROM %#RX64\n", pVmcb->guest.u64LASTEXCPFROM));
3952	Log4(("guest.u64LASTEXCPTO %#RX64\n", pVmcb->guest.u64LASTEXCPTO));
3953
3954	NOREF(pVmcb);
3955	#endif /* VBOX_STRICT */
3956	}
3957	else
3958	Log4Func(("rcVMRun=%d\n", rcVMRun));
3959	}
3960
3961
3962	/**
3963	* Check per-VM and per-VCPU force flag actions that require us to go back to
3964	* ring-3 for one reason or another.
3965	*
3966	* @returns Strict VBox status code (information status code included).
3967	* @retval VINF_SUCCESS if we don't have any actions that require going back to
3968	* ring-3.
3969	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
3970	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
3971	* interrupts)
3972	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
3973	* all EMTs to be in ring-3.
3974	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
3975	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
3976	* to the EM loop.
3977	*
3978	* @param pVCpu The cross context virtual CPU structure.
3979	*/
3980	static VBOXSTRICTRC hmR0SvmCheckForceFlags(PVMCPUCC pVCpu)
3981	{
3982	Assert(VMMRZCallRing3IsEnabled(pVCpu));
3983
3984	/* Could happen as a result of longjump. */
3985	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
3986	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
3987
3988	/* Update pending interrupts into the APIC's IRR. */
3989	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
3990	APICUpdatePendingInterrupts(pVCpu);
3991
3992	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3993	if ( VM_FF_IS_ANY_SET(pVM, !pVCpu->hm.s.fSingleInstruction
3994	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
3995	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, !pVCpu->hm.s.fSingleInstruction
3996	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
3997	{
3998	/* Pending PGM C3 sync. */
3999	if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
4000	{
4001	int rc = PGMSyncCR3(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.cr4,
4002	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
4003	if (rc != VINF_SUCCESS)
4004	{
4005	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
4006	return rc;
4007	}
4008	}
4009
4010	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
4011	/* -XXX- what was that about single stepping? */
4012	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
4013	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
4014	{
4015	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
4016	int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
4017	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
4018	return rc;
4019	}
4020
4021	/* Pending VM request packets, such as hardware interrupts. */
4022	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
4023	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
4024	{
4025	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchVmReq);
4026	Log4Func(("Pending VM request forcing us back to ring-3\n"));
4027	return VINF_EM_PENDING_REQUEST;
4028	}
4029
4030	/* Pending PGM pool flushes. */
4031	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
4032	{
4033	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPgmPoolFlush);
4034	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
4035	return VINF_PGM_POOL_FLUSH_PENDING;
4036	}
4037
4038	/* Pending DMA requests. */
4039	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
4040	{
4041	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchDma);
4042	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
4043	return VINF_EM_RAW_TO_R3;
4044	}
4045	}
4046
4047	return VINF_SUCCESS;
4048	}
4049
4050
4051	/**
4052	* Does the preparations before executing guest code in AMD-V.
4053	*
4054	* This may cause longjmps to ring-3 and may even result in rescheduling to the
4055	* recompiler. We must be cautious what we do here regarding committing
4056	* guest-state information into the VMCB assuming we assuredly execute the guest
4057	* in AMD-V. If we fall back to the recompiler after updating the VMCB and
4058	* clearing the common-state (TRPM/forceflags), we must undo those changes so
4059	* that the recompiler can (and should) use them when it resumes guest
4060	* execution. Otherwise such operations must be done when we can no longer
4061	* exit to ring-3.
4062	*
4063	* @returns Strict VBox status code (informational status codes included).
4064	* @retval VINF_SUCCESS if we can proceed with running the guest.
4065	* @retval VINF_* scheduling changes, we have to go back to ring-3.
4066	*
4067	* @param pVCpu The cross context virtual CPU structure.
4068	* @param pSvmTransient Pointer to the SVM transient structure.
4069	*/
4070	static VBOXSTRICTRC hmR0SvmPreRunGuest(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4071	{
4072	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
4073
4074	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
4075	if (pSvmTransient->fIsNestedGuest)
4076	{
4077	Log2(("hmR0SvmPreRunGuest: Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
4078	return VINF_EM_RESCHEDULE_REM;
4079	}
4080	#endif
4081
4082	/* Check force flag actions that might require us to go back to ring-3. */
4083	VBOXSTRICTRC rc = hmR0SvmCheckForceFlags(pVCpu);
4084	if (rc != VINF_SUCCESS)
4085	return rc;
4086
4087	if (TRPMHasTrap(pVCpu))
4088	hmR0SvmTrpmTrapToPendingEvent(pVCpu);
4089	else if (!pVCpu->hm.s.Event.fPending)
4090	{
4091	rc = hmR0SvmEvaluatePendingEvent(pVCpu, pSvmTransient);
4092	if ( rc != VINF_SUCCESS
4093	\|\| pSvmTransient->fIsNestedGuest != CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
4094	{
4095	/* If a nested-guest VM-exit occurred, bail. */
4096	if (pSvmTransient->fIsNestedGuest)
4097	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
4098	return rc;
4099	}
4100	}
4101
4102	/*
4103	* On the oldest AMD-V systems, we may not get enough information to reinject an NMI.
4104	* Just do it in software, see @bugref{8411}.
4105	* NB: If we could continue a task switch exit we wouldn't need to do this.
4106	*/
4107	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4108	if (RT_UNLIKELY( !g_fHmSvmFeatures
4109	&& pVCpu->hm.s.Event.fPending
4110	&& SVM_EVENT_GET_TYPE(pVCpu->hm.s.Event.u64IntInfo) == SVM_EVENT_NMI))
4111	return VINF_EM_RAW_INJECT_TRPM_EVENT;
4112
4113	#ifdef HMSVM_SYNC_FULL_GUEST_STATE
4114	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
4115	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
4116	#endif
4117
4118	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4119	/*
4120	* Set up the nested-guest VMCB for execution using hardware-assisted SVM.
4121	*/
4122	if (pSvmTransient->fIsNestedGuest)
4123	hmR0SvmSetupVmcbNested(pVCpu);
4124	#endif
4125
4126	/*
4127	* Export the guest state bits that are not shared with the host in any way as we can
4128	* longjmp or get preempted in the midst of exporting some of the state.
4129	*/
4130	rc = hmR0SvmExportGuestState(pVCpu, pSvmTransient);
4131	AssertRCReturn(rc, rc);
4132	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
4133
4134	/* Ensure we've cached (and hopefully modified) the nested-guest VMCB for execution using hardware-assisted SVM. */
4135	Assert(!pSvmTransient->fIsNestedGuest \|\| pVCpu->hm.s.svm.NstGstVmcbCache.fCacheValid);
4136
4137	/*
4138	* If we're not intercepting TPR changes in the guest, save the guest TPR before the
4139	* world-switch so we can update it on the way back if the guest changed the TPR.
4140	*/
4141	if (pVCpu->hmr0.s.svm.fSyncVTpr)
4142	{
4143	Assert(!pSvmTransient->fIsNestedGuest);
4144	PCSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
4145	if (pVM->hm.s.fTprPatchingActive)
4146	pSvmTransient->u8GuestTpr = pVmcb->guest.u64LSTAR;
4147	else
4148	pSvmTransient->u8GuestTpr = pVmcb->ctrl.IntCtrl.n.u8VTPR;
4149	}
4150
4151	/*
4152	* No longjmps to ring-3 from this point on!!!
4153	*
4154	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4155	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4156	*/
4157	VMMRZCallRing3Disable(pVCpu);
4158
4159	/*
4160	* We disable interrupts so that we don't miss any interrupts that would flag preemption
4161	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
4162	* preemption disabled for a while. Since this is purly to aid the
4163	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
4164	* disable interrupt on NT.
4165	*
4166	* We need to check for force-flags that could've possible been altered since we last
4167	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
4168	* see @bugref{6398}).
4169	*
4170	* We also check a couple of other force-flags as a last opportunity to get the EMT back
4171	* to ring-3 before executing guest code.
4172	*/
4173	pSvmTransient->fEFlags = ASMIntDisableFlags();
4174	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
4175	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
4176	{
4177	ASMSetFlags(pSvmTransient->fEFlags);
4178	VMMRZCallRing3Enable(pVCpu);
4179	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
4180	return VINF_EM_RAW_TO_R3;
4181	}
4182	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
4183	{
4184	ASMSetFlags(pSvmTransient->fEFlags);
4185	VMMRZCallRing3Enable(pVCpu);
4186	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
4187	return VINF_EM_RAW_INTERRUPT;
4188	}
4189
4190	return VINF_SUCCESS;
4191	}
4192
4193
4194	/**
4195	* Prepares to run guest (or nested-guest) code in AMD-V and we've committed to
4196	* doing so.
4197	*
4198	* This means there is no backing out to ring-3 or anywhere else at this point.
4199	*
4200	* @param pVCpu The cross context virtual CPU structure.
4201	* @param pSvmTransient Pointer to the SVM transient structure.
4202	*
4203	* @remarks Called with preemption disabled.
4204	* @remarks No-long-jump zone!!!
4205	*/
4206	static void hmR0SvmPreRunGuestCommitted(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4207	{
4208	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4209	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4210
4211	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4212	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
4213
4214	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4215	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
4216
4217	hmR0SvmInjectPendingEvent(pVCpu, pVmcb);
4218
4219	if (!CPUMIsGuestFPUStateActive(pVCpu))
4220	{
4221	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
4222	CPUMR0LoadGuestFPU(pVM, pVCpu); /* (Ignore rc, no need to set HM_CHANGED_HOST_CONTEXT for SVM.) */
4223	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
4224	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
4225	}
4226
4227	/* Load the state shared between host and guest (FPU, debug). */
4228	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)
4229	hmR0SvmExportSharedState(pVCpu, pVmcb);
4230
4231	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT; /* Preemption might set this, nothing to do on AMD-V. */
4232	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
4233
4234	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
4235	RTCPUID const idHostCpu = pHostCpu->idCpu;
4236	bool const fMigratedHostCpu = idHostCpu != pVCpu->hmr0.s.idLastCpu;
4237
4238	/* Setup TSC offsetting. */
4239	if ( pSvmTransient->fUpdateTscOffsetting
4240	\|\| fMigratedHostCpu)
4241	{
4242	hmR0SvmUpdateTscOffsetting(pVCpu, pVmcb);
4243	pSvmTransient->fUpdateTscOffsetting = false;
4244	}
4245
4246	/* Record statistics of how often we use TSC offsetting as opposed to intercepting RDTSC/P. */
4247	if (!(pVmcb->ctrl.u64InterceptCtrl & (SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP)))
4248	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4249	else
4250	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4251
4252	/* If we've migrating CPUs, mark the VMCB Clean bits as dirty. */
4253	if (fMigratedHostCpu)
4254	pVmcb->ctrl.u32VmcbCleanBits = 0;
4255
4256	/* Store status of the shared guest-host state at the time of VMRUN. */
4257	pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
4258	pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
4259
4260	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4261	uint8_t *pbMsrBitmap;
4262	if (!pSvmTransient->fIsNestedGuest)
4263	pbMsrBitmap = (uint8_t *)pVCpu->hmr0.s.svm.pvMsrBitmap;
4264	else
4265	{
4266	/** @todo We could perhaps optimize this by monitoring if the guest modifies its
4267	* MSRPM and only perform this if it changed also use EVEX.POR when it
4268	* does. */
4269	hmR0SvmMergeMsrpmNested(pHostCpu, pVCpu);
4270
4271	/* Update the nested-guest VMCB with the newly merged MSRPM (clean bits updated below). */
4272	pVmcb->ctrl.u64MSRPMPhysAddr = pHostCpu->n.svm.HCPhysNstGstMsrpm;
4273	pbMsrBitmap = (uint8_t *)pHostCpu->n.svm.pvNstGstMsrpm;
4274	}
4275	#else
4276	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.svm.pvMsrBitmap;
4277	#endif
4278
4279	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
4280	/* Flush the appropriate tagged-TLB entries. */
4281	hmR0SvmFlushTaggedTlb(pHostCpu, pVCpu, pVmcb);
4282	Assert(pVCpu->hmr0.s.idLastCpu == idHostCpu);
4283
4284	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
4285
4286	TMNotifyStartOfExecution(pVM, pVCpu); /* Finally, notify TM to resume its clocks as we're about
4287	to start executing. */
4288
4289	/*
4290	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that RDTSCPs
4291	* (that don't cause exits) reads the guest MSR, see @bugref{3324}.
4292	*
4293	* This should be done -after- any RDTSCPs for obtaining the host timestamp (TM, STAM etc).
4294	*/
4295	if ( g_CpumHostFeatures.s.fRdTscP
4296	&& !(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSCP))
4297	{
4298	uint64_t const uGuestTscAux = CPUMGetGuestTscAux(pVCpu);
4299	pVCpu->hmr0.s.svm.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
4300	if (uGuestTscAux != pVCpu->hmr0.s.svm.u64HostTscAux)
4301	ASMWrMsr(MSR_K8_TSC_AUX, uGuestTscAux);
4302	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
4303	pSvmTransient->fRestoreTscAuxMsr = true;
4304	}
4305	else
4306	{
4307	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
4308	pSvmTransient->fRestoreTscAuxMsr = false;
4309	}
4310	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
4311
4312	/*
4313	* If VMCB Clean bits isn't supported by the CPU or exposed to the guest in the nested
4314	* virtualization case, mark all state-bits as dirty indicating to the CPU to re-load
4315	* from the VMCB.
4316	*/
4317	bool const fSupportsVmcbCleanBits = hmR0SvmSupportsVmcbCleanBits(pVCpu, pSvmTransient->fIsNestedGuest);
4318	if (!fSupportsVmcbCleanBits)
4319	pVmcb->ctrl.u32VmcbCleanBits = 0;
4320	}
4321
4322
4323	/**
4324	* Wrapper for running the guest (or nested-guest) code in AMD-V.
4325	*
4326	* @returns VBox strict status code.
4327	* @param pVCpu The cross context virtual CPU structure.
4328	* @param HCPhysVmcb The host physical address of the VMCB.
4329	*
4330	* @remarks No-long-jump zone!!!
4331	*/
4332	DECLINLINE(int) hmR0SvmRunGuest(PVMCPUCC pVCpu, RTHCPHYS HCPhysVmcb)
4333	{
4334	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4335	pVCpu->cpum.GstCtx.fExtrn \|= HMSVM_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
4336	return pVCpu->hmr0.s.svm.pfnVMRun(pVCpu->CTX_SUFF(pVM), pVCpu, HCPhysVmcb);
4337	}
4338
4339
4340	/**
4341	* Performs some essential restoration of state after running guest (or
4342	* nested-guest) code in AMD-V.
4343	*
4344	* @param pVCpu The cross context virtual CPU structure.
4345	* @param pSvmTransient Pointer to the SVM transient structure.
4346	* @param rcVMRun Return code of VMRUN.
4347	*
4348	* @remarks Called with interrupts disabled.
4349	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
4350	* unconditionally when it is safe to do so.
4351	*/
4352	static void hmR0SvmPostRunGuest(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, VBOXSTRICTRC rcVMRun)
4353	{
4354	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4355
4356	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
4357	ASMAtomicIncU32(&pVCpu->hmr0.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
4358
4359	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
4360	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
4361
4362	/* TSC read must be done early for maximum accuracy. */
4363	if (!(pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSC))
4364	{
4365	if (!pSvmTransient->fIsNestedGuest)
4366	TMCpuTickSetLastSeen(pVCpu, pVCpu->hmr0.s.uTscExit + pVmcbCtrl->u64TSCOffset);
4367	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4368	else
4369	{
4370	/* The nested-guest VMCB TSC offset shall eventually be restored on #VMEXIT via HMNotifySvmNstGstVmexit(). */
4371	uint64_t const uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, pVCpu->hmr0.s.uTscExit + pVmcbCtrl->u64TSCOffset);
4372	TMCpuTickSetLastSeen(pVCpu, uGstTsc);
4373	}
4374	#endif
4375	}
4376
4377	if (pSvmTransient->fRestoreTscAuxMsr)
4378	{
4379	uint64_t u64GuestTscAuxMsr = ASMRdMsr(MSR_K8_TSC_AUX);
4380	CPUMSetGuestTscAux(pVCpu, u64GuestTscAuxMsr);
4381	if (u64GuestTscAuxMsr != pVCpu->hmr0.s.svm.u64HostTscAux)
4382	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hmr0.s.svm.u64HostTscAux);
4383	}
4384
4385	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
4386	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4387	TMNotifyEndOfExecution(pVM, pVCpu, pVCpu->hmr0.s.uTscExit); /* Notify TM that the guest is no longer running. */
4388	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4389
4390	Assert(!(ASMGetFlags() & X86_EFL_IF));
4391	ASMSetFlags(pSvmTransient->fEFlags); /* Enable interrupts. */
4392	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
4393
4394	/* If VMRUN failed, we can bail out early. This does -not- cover SVM_EXIT_INVALID. */
4395	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
4396	{
4397	Log4Func(("VMRUN failure: rcVMRun=%Rrc\n", VBOXSTRICTRC_VAL(rcVMRun)));
4398	return;
4399	}
4400
4401	pSvmTransient->u64ExitCode = pVmcbCtrl->u64ExitCode; /* Save the #VMEXIT reason. */
4402	pSvmTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
4403	pSvmTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
4404	pVmcbCtrl->u32VmcbCleanBits = HMSVM_VMCB_CLEAN_ALL; /* Mark the VMCB-state cache as unmodified by VMM. */
4405
4406	#ifdef HMSVM_SYNC_FULL_GUEST_STATE
4407	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4408	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
4409	#else
4410	/*
4411	* Always import the following:
4412	*
4413	* - RIP for exit optimizations and evaluating event injection on re-entry.
4414	* - RFLAGS for evaluating event injection on VM re-entry and for exporting shared debug
4415	* state on preemption.
4416	* - Interrupt shadow, GIF for evaluating event injection on VM re-entry.
4417	* - CS for exit optimizations.
4418	* - RAX, RSP for simplifying assumptions on GPRs. All other GPRs are swapped by the
4419	* assembly switcher code.
4420	* - Shared state (only DR7 currently) for exporting shared debug state on preemption.
4421	*/
4422	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
4423	\| CPUMCTX_EXTRN_RFLAGS
4424	\| CPUMCTX_EXTRN_RAX
4425	\| CPUMCTX_EXTRN_RSP
4426	\| CPUMCTX_EXTRN_CS
4427	\| CPUMCTX_EXTRN_HWVIRT
4428	\| CPUMCTX_EXTRN_INHIBIT_INT
4429	\| CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ
4430	\| HMSVM_CPUMCTX_SHARED_STATE);
4431	#endif
4432
4433	if ( pSvmTransient->u64ExitCode != SVM_EXIT_INVALID
4434	&& pVCpu->hmr0.s.svm.fSyncVTpr)
4435	{
4436	Assert(!pSvmTransient->fIsNestedGuest);
4437	/* TPR patching (for 32-bit guests) uses LSTAR MSR for holding the TPR value, otherwise uses the VTPR. */
4438	if ( pVM->hm.s.fTprPatchingActive
4439	&& (pVmcb->guest.u64LSTAR & 0xff) != pSvmTransient->u8GuestTpr)
4440	{
4441	int rc = APICSetTpr(pVCpu, pVmcb->guest.u64LSTAR & 0xff);
4442	AssertRC(rc);
4443	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
4444	}
4445	/* Sync TPR when we aren't intercepting CR8 writes. */
4446	else if (pSvmTransient->u8GuestTpr != pVmcbCtrl->IntCtrl.n.u8VTPR)
4447	{
4448	int rc = APICSetTpr(pVCpu, pVmcbCtrl->IntCtrl.n.u8VTPR << 4);
4449	AssertRC(rc);
4450	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
4451	}
4452	}
4453
4454	#ifdef DEBUG_ramshankar
4455	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
4456	{
4457	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4458	hmR0SvmLogState(pVCpu, pVmcb, pVCpu->cpum.GstCtx, "hmR0SvmPostRunGuestNested", HMSVM_LOG_ALL & ~HMSVM_LOG_LBR,
4459	0 /* uVerbose */);
4460	}
4461	#endif
4462
4463	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
4464	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_SVM, pSvmTransient->u64ExitCode & EMEXIT_F_TYPE_MASK),
4465	pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, pVCpu->hmr0.s.uTscExit);
4466	}
4467
4468
4469	/**
4470	* Runs the guest code using AMD-V.
4471	*
4472	* @returns Strict VBox status code.
4473	* @param pVCpu The cross context virtual CPU structure.
4474	* @param pcLoops Pointer to the number of executed loops.
4475	*/
4476	static VBOXSTRICTRC hmR0SvmRunGuestCodeNormal(PVMCPUCC pVCpu, uint32_t *pcLoops)
4477	{
4478	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops;
4479	Assert(pcLoops);
4480	Assert(*pcLoops <= cMaxResumeLoops);
4481
4482	SVMTRANSIENT SvmTransient;
4483	RT_ZERO(SvmTransient);
4484	SvmTransient.fUpdateTscOffsetting = true;
4485	SvmTransient.pVmcb = pVCpu->hmr0.s.svm.pVmcb;
4486
4487	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_5;
4488	for (;;)
4489	{
4490	Assert(!HMR0SuspendPending());
4491	HMSVM_ASSERT_CPU_SAFE(pVCpu);
4492
4493	/* Preparatory work for running nested-guest code, this may force us to return to
4494	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4495	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4496	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
4497	if (rc != VINF_SUCCESS)
4498	break;
4499
4500	/*
4501	* No longjmps to ring-3 from this point on!!!
4502	*
4503	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4504	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4505	*/
4506	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
4507	rc = hmR0SvmRunGuest(pVCpu, pVCpu->hmr0.s.svm.HCPhysVmcb);
4508
4509	/* Restore any residual host-state and save any bits shared between host and guest
4510	into the guest-CPU state. Re-enables interrupts! */
4511	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
4512
4513	if (RT_UNLIKELY( rc != VINF_SUCCESS /* Check for VMRUN errors. */
4514	\|\| SvmTransient.u64ExitCode == SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
4515	{
4516	if (rc == VINF_SUCCESS)
4517	rc = VERR_SVM_INVALID_GUEST_STATE;
4518	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
4519	hmR0SvmReportWorldSwitchError(pVCpu, VBOXSTRICTRC_VAL(rc));
4520	break;
4521	}
4522
4523	/* Handle the #VMEXIT. */
4524	HMSVM_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4525	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
4526	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, SvmTransient.u64ExitCode, pVCpu->hmr0.s.svm.pVmcb);
4527	rc = hmR0SvmHandleExit(pVCpu, &SvmTransient);
4528	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
4529	if (rc != VINF_SUCCESS)
4530	break;
4531	if (++(*pcLoops) >= cMaxResumeLoops)
4532	{
4533	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4534	rc = VINF_EM_RAW_INTERRUPT;
4535	break;
4536	}
4537	}
4538
4539	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4540	return rc;
4541	}
4542
4543
4544	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4545	/**
4546	* Runs the nested-guest code using AMD-V.
4547	*
4548	* @returns Strict VBox status code.
4549	* @param pVCpu The cross context virtual CPU structure.
4550	* @param pcLoops Pointer to the number of executed loops. If we're switching
4551	* from the guest-code execution loop to this nested-guest
4552	* execution loop pass the remainder value, else pass 0.
4553	*/
4554	static VBOXSTRICTRC hmR0SvmRunGuestCodeNested(PVMCPUCC pVCpu, uint32_t *pcLoops)
4555	{
4556	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4557	HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
4558	Assert(pcLoops);
4559	Assert(*pcLoops <= pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops);
4560	/** @todo r=bird: Sharing this with ring-3 isn't safe in the long run, I fear... */
4561	RTHCPHYS const HCPhysVmcb = GVMMR0ConvertGVMPtr2HCPhys(pVCpu->pGVM, &pCtx->hwvirt.svm.Vmcb);
4562
4563	SVMTRANSIENT SvmTransient;
4564	RT_ZERO(SvmTransient);
4565	SvmTransient.fUpdateTscOffsetting = true;
4566	SvmTransient.pVmcb = &pCtx->hwvirt.svm.Vmcb;
4567	SvmTransient.fIsNestedGuest = true;
4568
4569	/* Setup pointer so PGM/IEM can query #VMEXIT auxiliary info. on demand in ring-0. */
4570	pVCpu->hmr0.s.svm.pSvmTransient = &SvmTransient;
4571
4572	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_4;
4573	for (;;)
4574	{
4575	Assert(!HMR0SuspendPending());
4576	HMSVM_ASSERT_CPU_SAFE(pVCpu);
4577
4578	/* Preparatory work for running nested-guest code, this may force us to return to
4579	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4580	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4581	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
4582	if ( rc != VINF_SUCCESS
4583	\|\| !CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4584	break;
4585
4586	/*
4587	* No longjmps to ring-3 from this point on!!!
4588	*
4589	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4590	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4591	*/
4592	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
4593
4594	rc = hmR0SvmRunGuest(pVCpu, HCPhysVmcb);
4595
4596	/* Restore any residual host-state and save any bits shared between host and guest
4597	into the guest-CPU state. Re-enables interrupts! */
4598	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
4599
4600	if (RT_LIKELY( rc == VINF_SUCCESS
4601	&& SvmTransient.u64ExitCode != SVM_EXIT_INVALID))
4602	{ /* extremely likely */ }
4603	else
4604	{
4605	/* VMRUN failed, shouldn't really happen, Guru. */
4606	if (rc != VINF_SUCCESS)
4607	break;
4608
4609	/* Invalid nested-guest state. Cause a #VMEXIT but assert on strict builds. */
4610	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4611	AssertMsgFailed(("Invalid nested-guest state. rc=%Rrc u64ExitCode=%#RX64\n", rc, SvmTransient.u64ExitCode));
4612	rc = IEMExecSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0, 0);
4613	break;
4614	}
4615
4616	/* Handle the #VMEXIT. */
4617	HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4618	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
4619	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, &pCtx->hwvirt.svm.Vmcb);
4620	rc = hmR0SvmHandleExitNested(pVCpu, &SvmTransient);
4621	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
4622	if (rc == VINF_SUCCESS)
4623	{
4624	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4625	{
4626	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
4627	rc = VINF_SVM_VMEXIT;
4628	}
4629	else
4630	{
4631	if (++(*pcLoops) <= pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops)
4632	continue;
4633	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4634	rc = VINF_EM_RAW_INTERRUPT;
4635	}
4636	}
4637	else
4638	Assert(rc != VINF_SVM_VMEXIT);
4639	break;
4640	/** @todo NSTSVM: handle single-stepping. */
4641	}
4642
4643	/* Ensure #VMEXIT auxiliary info. is no longer available. */
4644	pVCpu->hmr0.s.svm.pSvmTransient = NULL;
4645
4646	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4647	return rc;
4648	}
4649	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
4650
4651
4652	/**
4653	* Checks if any expensive dtrace probes are enabled and we should go to the
4654	* debug loop.
4655	*
4656	* @returns true if we should use debug loop, false if not.
4657	*/
4658	static bool hmR0SvmAnyExpensiveProbesEnabled(void)
4659	{
4660	/* It's probably faster to OR the raw 32-bit counter variables together.
4661	Since the variables are in an array and the probes are next to one
4662	another (more or less), we have good locality. So, better read
4663	eight-nine cache lines ever time and only have one conditional, than
4664	128+ conditionals, right? */
4665	return ( VBOXVMM_R0_HMSVM_VMEXIT_ENABLED_RAW() /* expensive too due to context */
4666	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
4667	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
4668	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
4669	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
4670	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
4671	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
4672	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
4673	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
4674	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
4675	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
4676	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
4677	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
4678	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
4679	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
4680	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
4681	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
4682	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
4683	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
4684	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
4685	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
4686	) != 0
4687	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
4688	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
4689	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
4690	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
4691	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
4692	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
4693	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
4694	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
4695	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
4696	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
4697	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
4698	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
4699	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
4700	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
4701	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
4702	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
4703	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
4704	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
4705	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
4706	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
4707	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
4708	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
4709	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
4710	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
4711	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
4712	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
4713	//\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
4714	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
4715	//\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
4716	//\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
4717	//\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
4718	//\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
4719	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
4720	\| VBOXVMM_INSTR_SVM_VMRUN_ENABLED_RAW()
4721	\| VBOXVMM_INSTR_SVM_VMLOAD_ENABLED_RAW()
4722	\| VBOXVMM_INSTR_SVM_VMSAVE_ENABLED_RAW()
4723	\| VBOXVMM_INSTR_SVM_STGI_ENABLED_RAW()
4724	\| VBOXVMM_INSTR_SVM_CLGI_ENABLED_RAW()
4725	) != 0
4726	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
4727	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
4728	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
4729	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
4730	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
4731	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
4732	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
4733	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
4734	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
4735	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
4736	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
4737	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
4738	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
4739	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
4740	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
4741	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
4742	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
4743	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
4744	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
4745	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
4746	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
4747	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
4748	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
4749	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
4750	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
4751	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
4752	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
4753	//\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
4754	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
4755	//\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
4756	//\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
4757	//\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
4758	//\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
4759	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
4760	\| VBOXVMM_EXIT_SVM_VMRUN_ENABLED_RAW()
4761	\| VBOXVMM_EXIT_SVM_VMLOAD_ENABLED_RAW()
4762	\| VBOXVMM_EXIT_SVM_VMSAVE_ENABLED_RAW()
4763	\| VBOXVMM_EXIT_SVM_STGI_ENABLED_RAW()
4764	\| VBOXVMM_EXIT_SVM_CLGI_ENABLED_RAW()
4765	) != 0;
4766	}
4767
4768
4769	/**
4770	* Runs the guest code using AMD-V.
4771	*
4772	* @returns Strict VBox status code.
4773	* @param pVCpu The cross context virtual CPU structure.
4774	*/
4775	VMMR0DECL(VBOXSTRICTRC) SVMR0RunGuestCode(PVMCPUCC pVCpu)
4776	{
4777	AssertPtr(pVCpu);
4778	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4779	Assert(VMMRZCallRing3IsEnabled(pVCpu));
4780	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
4781	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
4782
4783	uint32_t cLoops = 0;
4784	VBOXSTRICTRC rc;
4785	for (;;)
4786	{
4787	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4788	bool const fInNestedGuestMode = CPUMIsGuestInSvmNestedHwVirtMode(pCtx);
4789	#else
4790	NOREF(pCtx);
4791	bool const fInNestedGuestMode = false;
4792	#endif
4793	if (!fInNestedGuestMode)
4794	{
4795	if ( !pVCpu->hm.s.fUseDebugLoop
4796	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0SvmAnyExpensiveProbesEnabled())
4797	&& !DBGFIsStepping(pVCpu)
4798	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
4799	rc = hmR0SvmRunGuestCodeNormal(pVCpu, &cLoops);
4800	else
4801	rc = hmR0SvmRunGuestCodeDebug(pVCpu, &cLoops);
4802	}
4803	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4804	else
4805	rc = hmR0SvmRunGuestCodeNested(pVCpu, &cLoops);
4806
4807	if (rc == VINF_SVM_VMRUN)
4808	{
4809	Assert(CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
4810	continue;
4811	}
4812	if (rc == VINF_SVM_VMEXIT)
4813	{
4814	Assert(!CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
4815	continue;
4816	}
4817	#endif
4818	break;
4819	}
4820
4821	/* Fixup error codes. */
4822	if (rc == VERR_EM_INTERPRETER)
4823	rc = VINF_EM_RAW_EMULATE_INSTR;
4824	else if (rc == VINF_EM_RESET)
4825	rc = VINF_EM_TRIPLE_FAULT;
4826
4827	/* Prepare to return to ring-3. This will remove longjmp notifications. */
4828	rc = hmR0SvmExitToRing3(pVCpu, rc);
4829	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
4830	Assert(!VMMR0AssertionIsNotificationSet(pVCpu));
4831	return rc;
4832	}
4833
4834	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4835
4836	/**
4837	* Determines whether the given I/O access should cause a nested-guest \#VMEXIT.
4838	*
4839	* @param pvIoBitmap Pointer to the nested-guest IO bitmap.
4840	* @param pIoExitInfo Pointer to the SVMIOIOEXITINFO.
4841	*/
4842	static bool hmR0SvmIsIoInterceptSet(void *pvIoBitmap, PSVMIOIOEXITINFO pIoExitInfo)
4843	{
4844	const uint16_t u16Port = pIoExitInfo->n.u16Port;
4845	const SVMIOIOTYPE enmIoType = (SVMIOIOTYPE)pIoExitInfo->n.u1Type;
4846	const uint8_t cbReg = (pIoExitInfo->u >> SVM_IOIO_OP_SIZE_SHIFT) & 7;
4847	const uint8_t cAddrSizeBits = ((pIoExitInfo->u >> SVM_IOIO_ADDR_SIZE_SHIFT) & 7) << 4;
4848	const uint8_t iEffSeg = pIoExitInfo->n.u3Seg;
4849	const bool fRep = pIoExitInfo->n.u1Rep;
4850	const bool fStrIo = pIoExitInfo->n.u1Str;
4851
4852	return CPUMIsSvmIoInterceptSet(pvIoBitmap, u16Port, enmIoType, cbReg, cAddrSizeBits, iEffSeg, fRep, fStrIo,
4853	NULL /* pIoExitInfo */);
4854	}
4855
4856
4857	/**
4858	* Handles a nested-guest \#VMEXIT (for all EXITCODE values except
4859	* SVM_EXIT_INVALID).
4860	*
4861	* @returns VBox status code (informational status codes included).
4862	* @param pVCpu The cross context virtual CPU structure.
4863	* @param pSvmTransient Pointer to the SVM transient structure.
4864	*/
4865	static VBOXSTRICTRC hmR0SvmHandleExitNested(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4866	{
4867	HMSVM_ASSERT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
4868	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
4869	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
4870
4871	/*
4872	* We import the complete state here because we use separate VMCBs for the guest and the
4873	* nested-guest, and the guest's VMCB is used after the #VMEXIT. We can only save/restore
4874	* the #VMEXIT specific state if we used the same VMCB for both guest and nested-guest.
4875	*/
4876	#define NST_GST_VMEXIT_CALL_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
4877	do { \
4878	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL); \
4879	return IEMExecSvmVmexit((a_pVCpu), (a_uExitCode), (a_uExitInfo1), (a_uExitInfo2)); \
4880	} while (0)
4881
4882	/*
4883	* For all the #VMEXITs here we primarily figure out if the #VMEXIT is expected by the
4884	* nested-guest. If it isn't, it should be handled by the (outer) guest.
4885	*/
4886	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
4887	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4888	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
4889	uint64_t const uExitCode = pVmcbNstGstCtrl->u64ExitCode;
4890	uint64_t const uExitInfo1 = pVmcbNstGstCtrl->u64ExitInfo1;
4891	uint64_t const uExitInfo2 = pVmcbNstGstCtrl->u64ExitInfo2;
4892
4893	Assert(uExitCode == pVmcbNstGstCtrl->u64ExitCode);
4894	switch (uExitCode)
4895	{
4896	case SVM_EXIT_CPUID:
4897	{
4898	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CPUID))
4899	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4900	return hmR0SvmExitCpuid(pVCpu, pSvmTransient);
4901	}
4902
4903	case SVM_EXIT_RDTSC:
4904	{
4905	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSC))
4906	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4907	return hmR0SvmExitRdtsc(pVCpu, pSvmTransient);
4908	}
4909
4910	case SVM_EXIT_RDTSCP:
4911	{
4912	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSCP))
4913	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4914	return hmR0SvmExitRdtscp(pVCpu, pSvmTransient);
4915	}
4916
4917	case SVM_EXIT_MONITOR:
4918	{
4919	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MONITOR))
4920	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4921	return hmR0SvmExitMonitor(pVCpu, pSvmTransient);
4922	}
4923
4924	case SVM_EXIT_MWAIT:
4925	{
4926	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MWAIT))
4927	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4928	return hmR0SvmExitMwait(pVCpu, pSvmTransient);
4929	}
4930
4931	case SVM_EXIT_HLT:
4932	{
4933	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_HLT))
4934	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4935	return hmR0SvmExitHlt(pVCpu, pSvmTransient);
4936	}
4937
4938	case SVM_EXIT_MSR:
4939	{
4940	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MSR_PROT))
4941	{
4942	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
4943	uint16_t offMsrpm;
4944	uint8_t uMsrpmBit;
4945	int rc = CPUMGetSvmMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
4946	if (RT_SUCCESS(rc))
4947	{
4948	Assert(uMsrpmBit == 0 \|\| uMsrpmBit == 2 \|\| uMsrpmBit == 4 \|\| uMsrpmBit == 6);
4949	Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
4950
4951	uint8_t const * const pbMsrBitmap = &pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm];
4952	bool const fInterceptRead = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit));
4953	bool const fInterceptWrite = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
4954
4955	if ( (fInterceptWrite && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_WRITE)
4956	\|\| (fInterceptRead && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_READ))
4957	{
4958	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4959	}
4960	}
4961	else
4962	{
4963	/*
4964	* MSRs not covered by the MSRPM automatically cause an #VMEXIT.
4965	* See AMD-V spec. "15.11 MSR Intercepts".
4966	*/
4967	Assert(rc == VERR_OUT_OF_RANGE);
4968	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4969	}
4970	}
4971	return hmR0SvmExitMsr(pVCpu, pSvmTransient);
4972	}
4973
4974	case SVM_EXIT_IOIO:
4975	{
4976	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IOIO_PROT))
4977	{
4978	SVMIOIOEXITINFO IoExitInfo;
4979	IoExitInfo.u = pVmcbNstGst->ctrl.u64ExitInfo1;
4980	bool const fIntercept = hmR0SvmIsIoInterceptSet(pVCpu->cpum.GstCtx.hwvirt.svm.abIoBitmap, &IoExitInfo);
4981	if (fIntercept)
4982	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4983	}
4984	return hmR0SvmExitIOInstr(pVCpu, pSvmTransient);
4985	}
4986
4987	case SVM_EXIT_XCPT_PF:
4988	{
4989	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4990	if (pVM->hmr0.s.fNestedPaging)
4991	{
4992	uint32_t const u32ErrCode = pVmcbNstGstCtrl->u64ExitInfo1;
4993	uint64_t const uFaultAddress = pVmcbNstGstCtrl->u64ExitInfo2;
4994
4995	/* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
4996	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_PF))
4997	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, u32ErrCode, uFaultAddress);
4998
4999	/* If the nested-guest is not intercepting #PFs, forward the #PF to the guest. */
5000	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR2);
5001	hmR0SvmSetPendingXcptPF(pVCpu, u32ErrCode, uFaultAddress);
5002	return VINF_SUCCESS;
5003	}
5004	return hmR0SvmExitXcptPF(pVCpu, pSvmTransient);
5005	}
5006
5007	case SVM_EXIT_XCPT_UD:
5008	{
5009	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_UD))
5010	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5011	hmR0SvmSetPendingXcptUD(pVCpu);
5012	return VINF_SUCCESS;
5013	}
5014
5015	case SVM_EXIT_XCPT_MF:
5016	{
5017	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_MF))
5018	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5019	return hmR0SvmExitXcptMF(pVCpu, pSvmTransient);
5020	}
5021
5022	case SVM_EXIT_XCPT_DB:
5023	{
5024	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_DB))
5025	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5026	return hmR0SvmNestedExitXcptDB(pVCpu, pSvmTransient);
5027	}
5028
5029	case SVM_EXIT_XCPT_AC:
5030	{
5031	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_AC))
5032	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5033	return hmR0SvmExitXcptAC(pVCpu, pSvmTransient);
5034	}
5035
5036	case SVM_EXIT_XCPT_BP:
5037	{
5038	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_BP))
5039	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5040	return hmR0SvmNestedExitXcptBP(pVCpu, pSvmTransient);
5041	}
5042
5043	case SVM_EXIT_READ_CR0:
5044	case SVM_EXIT_READ_CR3:
5045	case SVM_EXIT_READ_CR4:
5046	{
5047	uint8_t const uCr = uExitCode - SVM_EXIT_READ_CR0;
5048	if (CPUMIsGuestSvmReadCRxInterceptSet(pVCpu, pCtx, uCr))
5049	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5050	return hmR0SvmExitReadCRx(pVCpu, pSvmTransient);
5051	}
5052
5053	case SVM_EXIT_CR0_SEL_WRITE:
5054	{
5055	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CR0_SEL_WRITE))
5056	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5057	return hmR0SvmExitWriteCRx(pVCpu, pSvmTransient);
5058	}
5059
5060	case SVM_EXIT_WRITE_CR0:
5061	case SVM_EXIT_WRITE_CR3:
5062	case SVM_EXIT_WRITE_CR4:
5063	case SVM_EXIT_WRITE_CR8: /* CR8 writes would go to the V_TPR rather than here, since we run with V_INTR_MASKING. */
5064	{
5065	uint8_t const uCr = uExitCode - SVM_EXIT_WRITE_CR0;
5066	Log4Func(("Write CR%u: uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", uCr, uExitInfo1, uExitInfo2));
5067
5068	if (CPUMIsGuestSvmWriteCRxInterceptSet(pVCpu, pCtx, uCr))
5069	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5070	return hmR0SvmExitWriteCRx(pVCpu, pSvmTransient);
5071	}
5072
5073	case SVM_EXIT_PAUSE:
5074	{
5075	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_PAUSE))
5076	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5077	return hmR0SvmExitPause(pVCpu, pSvmTransient);
5078	}
5079
5080	case SVM_EXIT_VINTR:
5081	{
5082	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VINTR))
5083	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5084	return hmR0SvmExitUnexpected(pVCpu, pSvmTransient);
5085	}
5086
5087	case SVM_EXIT_INTR:
5088	case SVM_EXIT_NMI:
5089	case SVM_EXIT_SMI:
5090	case SVM_EXIT_XCPT_NMI: /* Should not occur, SVM_EXIT_NMI is used instead. */
5091	{
5092	/*
5093	* We shouldn't direct physical interrupts, NMIs, SMIs to the nested-guest.
5094	*
5095	* Although we don't intercept SMIs, the nested-guest might. Therefore, we might
5096	* get an SMI #VMEXIT here so simply ignore rather than causing a corresponding
5097	* nested-guest #VMEXIT.
5098	*
5099	* We shall import the complete state here as we may cause #VMEXITs from ring-3
5100	* while trying to inject interrupts, see comment at the top of this function.
5101	*/
5102	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_ALL);
5103	return hmR0SvmExitIntr(pVCpu, pSvmTransient);
5104	}
5105
5106	case SVM_EXIT_FERR_FREEZE:
5107	{
5108	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_FERR_FREEZE))
5109	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5110	return hmR0SvmExitFerrFreeze(pVCpu, pSvmTransient);
5111	}
5112
5113	case SVM_EXIT_INVLPG:
5114	{
5115	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPG))
5116	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5117	return hmR0SvmExitInvlpg(pVCpu, pSvmTransient);
5118	}
5119
5120	case SVM_EXIT_WBINVD:
5121	{
5122	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_WBINVD))
5123	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5124	return hmR0SvmExitWbinvd(pVCpu, pSvmTransient);
5125	}
5126
5127	case SVM_EXIT_INVD:
5128	{
5129	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVD))
5130	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5131	return hmR0SvmExitInvd(pVCpu, pSvmTransient);
5132	}
5133
5134	case SVM_EXIT_RDPMC:
5135	{
5136	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDPMC))
5137	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5138	return hmR0SvmExitRdpmc(pVCpu, pSvmTransient);
5139	}
5140
5141	default:
5142	{
5143	switch (uExitCode)
5144	{
5145	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5146	case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5147	case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5148	case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5149	{
5150	uint8_t const uDr = uExitCode - SVM_EXIT_READ_DR0;
5151	if (CPUMIsGuestSvmReadDRxInterceptSet(pVCpu, pCtx, uDr))
5152	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5153	return hmR0SvmExitReadDRx(pVCpu, pSvmTransient);
5154	}
5155
5156	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5157	case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5158	case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5159	case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5160	{
5161	uint8_t const uDr = uExitCode - SVM_EXIT_WRITE_DR0;
5162	if (CPUMIsGuestSvmWriteDRxInterceptSet(pVCpu, pCtx, uDr))
5163	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5164	return hmR0SvmExitWriteDRx(pVCpu, pSvmTransient);
5165	}
5166
5167	case SVM_EXIT_XCPT_DE:
5168	/* SVM_EXIT_XCPT_DB: / / Handled above. */
5169	/* SVM_EXIT_XCPT_NMI: / / Handled above. */
5170	/* SVM_EXIT_XCPT_BP: / / Handled above. */
5171	case SVM_EXIT_XCPT_OF:
5172	case SVM_EXIT_XCPT_BR:
5173	/* SVM_EXIT_XCPT_UD: / / Handled above. */
5174	case SVM_EXIT_XCPT_NM:
5175	case SVM_EXIT_XCPT_DF:
5176	case SVM_EXIT_XCPT_CO_SEG_OVERRUN:
5177	case SVM_EXIT_XCPT_TS:
5178	case SVM_EXIT_XCPT_NP:
5179	case SVM_EXIT_XCPT_SS:
5180	case SVM_EXIT_XCPT_GP:
5181	/* SVM_EXIT_XCPT_PF: / / Handled above. */
5182	case SVM_EXIT_XCPT_15: /* Reserved. */
5183	/* SVM_EXIT_XCPT_MF: / / Handled above. */
5184	/* SVM_EXIT_XCPT_AC: / / Handled above. */
5185	case SVM_EXIT_XCPT_MC:
5186	case SVM_EXIT_XCPT_XF:
5187	case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
5188	case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
5189	case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
5190	{
5191	uint8_t const uVector = uExitCode - SVM_EXIT_XCPT_0;
5192	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, uVector))
5193	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5194	return hmR0SvmExitXcptGeneric(pVCpu, pSvmTransient);
5195	}
5196
5197	case SVM_EXIT_XSETBV:
5198	{
5199	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_XSETBV))
5200	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5201	return hmR0SvmExitXsetbv(pVCpu, pSvmTransient);
5202	}
5203
5204	case SVM_EXIT_TASK_SWITCH:
5205	{
5206	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_TASK_SWITCH))
5207	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5208	return hmR0SvmExitTaskSwitch(pVCpu, pSvmTransient);
5209	}
5210
5211	case SVM_EXIT_IRET:
5212	{
5213	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IRET))
5214	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5215	return hmR0SvmExitIret(pVCpu, pSvmTransient);
5216	}
5217
5218	case SVM_EXIT_SHUTDOWN:
5219	{
5220	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SHUTDOWN))
5221	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5222	return hmR0SvmExitShutdown(pVCpu, pSvmTransient);
5223	}
5224
5225	case SVM_EXIT_VMMCALL:
5226	{
5227	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMMCALL))
5228	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5229	return hmR0SvmExitVmmCall(pVCpu, pSvmTransient);
5230	}
5231
5232	case SVM_EXIT_CLGI:
5233	{
5234	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CLGI))
5235	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5236	return hmR0SvmExitClgi(pVCpu, pSvmTransient);
5237	}
5238
5239	case SVM_EXIT_STGI:
5240	{
5241	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_STGI))
5242	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5243	return hmR0SvmExitStgi(pVCpu, pSvmTransient);
5244	}
5245
5246	case SVM_EXIT_VMLOAD:
5247	{
5248	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMLOAD))
5249	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5250	return hmR0SvmExitVmload(pVCpu, pSvmTransient);
5251	}
5252
5253	case SVM_EXIT_VMSAVE:
5254	{
5255	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMSAVE))
5256	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5257	return hmR0SvmExitVmsave(pVCpu, pSvmTransient);
5258	}
5259
5260	case SVM_EXIT_INVLPGA:
5261	{
5262	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPGA))
5263	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5264	return hmR0SvmExitInvlpga(pVCpu, pSvmTransient);
5265	}
5266
5267	case SVM_EXIT_VMRUN:
5268	{
5269	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMRUN))
5270	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5271	return hmR0SvmExitVmrun(pVCpu, pSvmTransient);
5272	}
5273
5274	case SVM_EXIT_RSM:
5275	{
5276	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RSM))
5277	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5278	hmR0SvmSetPendingXcptUD(pVCpu);
5279	return VINF_SUCCESS;
5280	}
5281
5282	case SVM_EXIT_SKINIT:
5283	{
5284	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SKINIT))
5285	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5286	hmR0SvmSetPendingXcptUD(pVCpu);
5287	return VINF_SUCCESS;
5288	}
5289
5290	case SVM_EXIT_NPF:
5291	{
5292	Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
5293	return hmR0SvmExitNestedPF(pVCpu, pSvmTransient);
5294	}
5295
5296	case SVM_EXIT_INIT: /* We shouldn't get INIT signals while executing a nested-guest. */
5297	return hmR0SvmExitUnexpected(pVCpu, pSvmTransient);
5298
5299	default:
5300	{
5301	AssertMsgFailed(("hmR0SvmHandleExitNested: Unknown exit code %#x\n", pSvmTransient->u64ExitCode));
5302	pVCpu->hm.s.u32HMError = pSvmTransient->u64ExitCode;
5303	return VERR_SVM_UNKNOWN_EXIT;
5304	}
5305	}
5306	}
5307	}
5308	/* not reached */
5309
5310	# undef NST_GST_VMEXIT_CALL_RET
5311	}
5312
5313	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
5314
5315	/** @def VMEXIT_CALL_RET
5316	* Used by hmR0SvmHandleExit and hmR0SvmDebugHandleExit
5317	*/
5318	#ifdef DEBUG_ramshankar
5319	# define VMEXIT_CALL_RET(a_fDbg, a_CallExpr) \
5320	do { \
5321	if ((a_fDbg) == 1) \
5322	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL); \
5323	int rc = a_CallExpr; \
5324	if ((a_fDbg) == 1) \
5325	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
5326	return rc; \
5327	} while (0)
5328	#else
5329	# define VMEXIT_CALL_RET(a_fDbg, a_CallExpr) return a_CallExpr
5330	#endif
5331
5332	/**
5333	* Handles a guest \#VMEXIT (for all EXITCODE values except SVM_EXIT_INVALID).
5334	*
5335	* @returns Strict VBox status code (informational status codes included).
5336	* @param pVCpu The cross context virtual CPU structure.
5337	* @param pSvmTransient Pointer to the SVM transient structure.
5338	*/
5339	static VBOXSTRICTRC hmR0SvmHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
5340	{
5341	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
5342	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
5343
5344	/*
5345	* The ordering of the case labels is based on most-frequently-occurring #VMEXITs
5346	* for most guests under normal workloads (for some definition of "normal").
5347	*/
5348	uint64_t const uExitCode = pSvmTransient->u64ExitCode;
5349	switch (uExitCode)
5350	{
5351	case SVM_EXIT_NPF: VMEXIT_CALL_RET(0, hmR0SvmExitNestedPF(pVCpu, pSvmTransient));
5352	case SVM_EXIT_IOIO: VMEXIT_CALL_RET(0, hmR0SvmExitIOInstr(pVCpu, pSvmTransient));
5353	case SVM_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0SvmExitRdtsc(pVCpu, pSvmTransient));
5354	case SVM_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0SvmExitRdtscp(pVCpu, pSvmTransient));
5355	case SVM_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0SvmExitCpuid(pVCpu, pSvmTransient));
5356	case SVM_EXIT_XCPT_PF: VMEXIT_CALL_RET(0, hmR0SvmExitXcptPF(pVCpu, pSvmTransient));
5357	case SVM_EXIT_MSR: VMEXIT_CALL_RET(0, hmR0SvmExitMsr(pVCpu, pSvmTransient));
5358	case SVM_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0SvmExitMonitor(pVCpu, pSvmTransient));
5359	case SVM_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0SvmExitMwait(pVCpu, pSvmTransient));
5360	case SVM_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0SvmExitHlt(pVCpu, pSvmTransient));
5361
5362	case SVM_EXIT_XCPT_NMI: /* Should not occur, SVM_EXIT_NMI is used instead. */
5363	case SVM_EXIT_INTR:
5364	case SVM_EXIT_NMI: VMEXIT_CALL_RET(0, hmR0SvmExitIntr(pVCpu, pSvmTransient));
5365
5366	case SVM_EXIT_READ_CR0:
5367	case SVM_EXIT_READ_CR3:
5368	case SVM_EXIT_READ_CR4: VMEXIT_CALL_RET(0, hmR0SvmExitReadCRx(pVCpu, pSvmTransient));
5369
5370	case SVM_EXIT_CR0_SEL_WRITE:
5371	case SVM_EXIT_WRITE_CR0:
5372	case SVM_EXIT_WRITE_CR3:
5373	case SVM_EXIT_WRITE_CR4:
5374	case SVM_EXIT_WRITE_CR8: VMEXIT_CALL_RET(0, hmR0SvmExitWriteCRx(pVCpu, pSvmTransient));
5375
5376	case SVM_EXIT_VINTR: VMEXIT_CALL_RET(0, hmR0SvmExitVIntr(pVCpu, pSvmTransient));
5377	case SVM_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0SvmExitPause(pVCpu, pSvmTransient));
5378	case SVM_EXIT_VMMCALL: VMEXIT_CALL_RET(0, hmR0SvmExitVmmCall(pVCpu, pSvmTransient));
5379	case SVM_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0SvmExitInvlpg(pVCpu, pSvmTransient));
5380	case SVM_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0SvmExitWbinvd(pVCpu, pSvmTransient));
5381	case SVM_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0SvmExitInvd(pVCpu, pSvmTransient));
5382	case SVM_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0SvmExitRdpmc(pVCpu, pSvmTransient));
5383	case SVM_EXIT_IRET: VMEXIT_CALL_RET(0, hmR0SvmExitIret(pVCpu, pSvmTransient));
5384	case SVM_EXIT_XCPT_DE: VMEXIT_CALL_RET(0, hmR0SvmExitXcptDE(pVCpu, pSvmTransient));
5385	case SVM_EXIT_XCPT_UD: VMEXIT_CALL_RET(0, hmR0SvmExitXcptUD(pVCpu, pSvmTransient));
5386	case SVM_EXIT_XCPT_MF: VMEXIT_CALL_RET(0, hmR0SvmExitXcptMF(pVCpu, pSvmTransient));
5387	case SVM_EXIT_XCPT_DB: VMEXIT_CALL_RET(0, hmR0SvmExitXcptDB(pVCpu, pSvmTransient));
5388	case SVM_EXIT_XCPT_AC: VMEXIT_CALL_RET(0, hmR0SvmExitXcptAC(pVCpu, pSvmTransient));
5389	case SVM_EXIT_XCPT_BP: VMEXIT_CALL_RET(0, hmR0SvmExitXcptBP(pVCpu, pSvmTransient));
5390	case SVM_EXIT_XCPT_GP: VMEXIT_CALL_RET(0, hmR0SvmExitXcptGP(pVCpu, pSvmTransient));
5391	case SVM_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0SvmExitXsetbv(pVCpu, pSvmTransient));
5392	case SVM_EXIT_FERR_FREEZE: VMEXIT_CALL_RET(0, hmR0SvmExitFerrFreeze(pVCpu, pSvmTransient));
5393
5394	default:
5395	{
5396	switch (pSvmTransient->u64ExitCode)
5397	{
5398	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5399	case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5400	case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5401	case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5402	VMEXIT_CALL_RET(0, hmR0SvmExitReadDRx(pVCpu, pSvmTransient));
5403
5404	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5405	case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5406	case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5407	case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5408	VMEXIT_CALL_RET(0, hmR0SvmExitWriteDRx(pVCpu, pSvmTransient));
5409
5410	case SVM_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0SvmExitTaskSwitch(pVCpu, pSvmTransient));
5411	case SVM_EXIT_SHUTDOWN: VMEXIT_CALL_RET(0, hmR0SvmExitShutdown(pVCpu, pSvmTransient));
5412
5413	case SVM_EXIT_SMI:
5414	case SVM_EXIT_INIT:
5415	{
5416	/*
5417	* We don't intercept SMIs. As for INIT signals, it really shouldn't ever happen here.
5418	* If it ever does, we want to know about it so log the exit code and bail.
5419	*/
5420	VMEXIT_CALL_RET(0, hmR0SvmExitUnexpected(pVCpu, pSvmTransient));
5421	}
5422
5423	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
5424	case SVM_EXIT_CLGI: VMEXIT_CALL_RET(0, hmR0SvmExitClgi(pVCpu, pSvmTransient));
5425	case SVM_EXIT_STGI: VMEXIT_CALL_RET(0, hmR0SvmExitStgi(pVCpu, pSvmTransient));
5426	case SVM_EXIT_VMLOAD: VMEXIT_CALL_RET(0, hmR0SvmExitVmload(pVCpu, pSvmTransient));
5427	case SVM_EXIT_VMSAVE: VMEXIT_CALL_RET(0, hmR0SvmExitVmsave(pVCpu, pSvmTransient));
5428	case SVM_EXIT_INVLPGA: VMEXIT_CALL_RET(0, hmR0SvmExitInvlpga(pVCpu, pSvmTransient));
5429	case SVM_EXIT_VMRUN: VMEXIT_CALL_RET(0, hmR0SvmExitVmrun(pVCpu, pSvmTransient));
5430	#else
5431	case SVM_EXIT_CLGI:
5432	case SVM_EXIT_STGI:
5433	case SVM_EXIT_VMLOAD:
5434	case SVM_EXIT_VMSAVE:
5435	case SVM_EXIT_INVLPGA:
5436	case SVM_EXIT_VMRUN:
5437	#endif
5438	case SVM_EXIT_RSM:
5439	case SVM_EXIT_SKINIT:
5440	{
5441	hmR0SvmSetPendingXcptUD(pVCpu);
5442	return VINF_SUCCESS;
5443	}
5444
5445	/*
5446	* The remaining should only be possible when debugging or dtracing.
5447	*/
5448	case SVM_EXIT_XCPT_DE:
5449	/* SVM_EXIT_XCPT_DB: / / Handled above. */
5450	/* SVM_EXIT_XCPT_NMI: / / Handled above. */
5451	/* SVM_EXIT_XCPT_BP: / / Handled above. */
5452	case SVM_EXIT_XCPT_OF:
5453	case SVM_EXIT_XCPT_BR:
5454	/* SVM_EXIT_XCPT_UD: / / Handled above. */
5455	case SVM_EXIT_XCPT_NM:
5456	case SVM_EXIT_XCPT_DF:
5457	case SVM_EXIT_XCPT_CO_SEG_OVERRUN:
5458	case SVM_EXIT_XCPT_TS:
5459	case SVM_EXIT_XCPT_NP:
5460	case SVM_EXIT_XCPT_SS:
5461	/* SVM_EXIT_XCPT_GP: / / Handled above. */
5462	/* SVM_EXIT_XCPT_PF: */
5463	case SVM_EXIT_XCPT_15: /* Reserved. */
5464	/* SVM_EXIT_XCPT_MF: / / Handled above. */
5465	/* SVM_EXIT_XCPT_AC: / / Handled above. */
5466	case SVM_EXIT_XCPT_MC:
5467	case SVM_EXIT_XCPT_XF:
5468	case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
5469	case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
5470	case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
5471	VMEXIT_CALL_RET(0, hmR0SvmExitXcptGeneric(pVCpu, pSvmTransient));
5472
5473	case SVM_EXIT_SWINT: VMEXIT_CALL_RET(0, hmR0SvmExitSwInt(pVCpu, pSvmTransient));
5474	case SVM_EXIT_TR_READ: VMEXIT_CALL_RET(0, hmR0SvmExitTrRead(pVCpu, pSvmTransient));
5475	case SVM_EXIT_TR_WRITE: VMEXIT_CALL_RET(0, hmR0SvmExitTrWrite(pVCpu, pSvmTransient)); /* Also OS/2 TLB workaround. */
5476
5477	default:
5478	{
5479	AssertMsgFailed(("hmR0SvmHandleExit: Unknown exit code %#RX64\n", uExitCode));
5480	pVCpu->hm.s.u32HMError = uExitCode;
5481	return VERR_SVM_UNKNOWN_EXIT;
5482	}
5483	}
5484	}
5485	}
5486	/* not reached */
5487	}
5488
5489
5490	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace probes.
5491	*
5492	* The following few functions and associated structure contains the bloat
5493	* necessary for providing detailed debug events and dtrace probes as well as
5494	* reliable host side single stepping. This works on the principle of
5495	* "subclassing" the normal execution loop and workers. We replace the loop
5496	* method completely and override selected helpers to add necessary adjustments
5497	* to their core operation.
5498	*
5499	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
5500	* any performance for debug and analysis features.
5501	*
5502	* @{
5503	*/
5504
5505	/**
5506	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
5507	* the debug run loop.
5508	*/
5509	typedef struct SVMRUNDBGSTATE
5510	{
5511	/** The initial SVMVMCBCTRL::u64InterceptCtrl value (helps with restore). */
5512	uint64_t bmInterceptInitial;
5513	/** The initial SVMVMCBCTRL::u32InterceptXcpt value (helps with restore). */
5514	uint32_t bmXcptInitial;
5515	/** The initial SVMVMCBCTRL::u16InterceptRdCRx value (helps with restore). */
5516	uint16_t bmInterceptRdCRxInitial;
5517	/** The initial SVMVMCBCTRL::u16InterceptWrCRx value (helps with restore). */
5518	uint16_t bmInterceptWrCRxInitial;
5519	/** The initial SVMVMCBCTRL::u16InterceptRdDRx value (helps with restore). */
5520	uint16_t bmInterceptRdDRxInitial;
5521	/** The initial SVMVMCBCTRL::u16InterceptWrDRx value (helps with restore). */
5522	uint16_t bmInterceptWrDRxInitial;
5523
5524	/** Whether we've actually modified the intercept control qword. */
5525	bool fModifiedInterceptCtrl : 1;
5526	/** Whether we've actually modified the exception bitmap. */
5527	bool fModifiedXcptBitmap : 1;
5528	/** Whether we've actually modified SVMVMCBCTRL::u16InterceptRdCRx. */
5529	bool fModifiedInterceptRdCRx : 1;
5530	/** Whether we've actually modified SVMVMCBCTRL::u16InterceptWrCRx. */
5531	bool fModifiedInterceptWrCRx : 1;
5532	/** Whether we've actually modified SVMVMCBCTRL::u16InterceptRdDRx. */
5533	bool fModifiedInterceptRdDRx : 1;
5534	/** Whether we've actually modified SVMVMCBCTRL::u16InterceptWrDRx. */
5535	bool fModifiedInterceptWrDRx : 1;
5536
5537	/** The CS we started executing with. */
5538	uint16_t uCsStart;
5539	/** The RIP we started executing at. This is for detecting that we stepped. */
5540	uint64_t uRipStart;
5541
5542	/** The sequence number of the Dtrace provider settings the state was
5543	* configured against. */
5544	uint32_t uDtraceSettingsSeqNo;
5545	/** Extra stuff we need in SVMVMCBCTRL::u32InterceptXcpt. */
5546	uint32_t bmXcptExtra;
5547	/** Extra stuff we need in SVMVMCBCTRL::u64InterceptCtrl. */
5548	uint64_t bmInterceptExtra;
5549	/** Extra stuff we need in SVMVMCBCTRL::u16InterceptRdCRx. */
5550	uint16_t bmInterceptRdCRxExtra;
5551	/** Extra stuff we need in SVMVMCBCTRL::u16InterceptWrCRx. */
5552	uint16_t bmInterceptWrCRxExtra;
5553	/** Extra stuff we need in SVMVMCBCTRL::u16InterceptRdDRx. */
5554	uint16_t bmInterceptRdDRxExtra;
5555	/** Extra stuff we need in SVMVMCBCTRL::u16InterceptWrDRx. */
5556	uint16_t bmInterceptWrDRxExtra;
5557	/** VM-exits to check (one bit per VM-exit). */
5558	uint32_t bmExitsToCheck[33];
5559	} SVMRUNDBGSTATE;
5560	AssertCompileMemberSize(SVMRUNDBGSTATE, bmExitsToCheck, (SVM_EXIT_MAX + 1 + 31) / 32 * 4);
5561	typedef SVMRUNDBGSTATE *PSVMRUNDBGSTATE;
5562
5563
5564	/**
5565	* Initializes the SVMRUNDBGSTATE structure.
5566	*
5567	* @param pVCpu The cross context virtual CPU structure of the
5568	* calling EMT.
5569	* @param pSvmTransient The SVM-transient structure.
5570	* @param pDbgState The debug state to initialize.
5571	*/
5572	static void hmR0SvmRunDebugStateInit(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5573	{
5574	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
5575	pDbgState->bmInterceptInitial = pVmcb->ctrl.u64InterceptCtrl;
5576	pDbgState->bmXcptInitial = pVmcb->ctrl.u32InterceptXcpt;
5577	pDbgState->bmInterceptRdCRxInitial = pVmcb->ctrl.u16InterceptRdCRx;
5578	pDbgState->bmInterceptWrCRxInitial = pVmcb->ctrl.u16InterceptWrCRx;
5579	pDbgState->bmInterceptRdDRxInitial = pVmcb->ctrl.u16InterceptRdDRx;
5580	pDbgState->bmInterceptWrDRxInitial = pVmcb->ctrl.u16InterceptWrDRx;
5581
5582	pDbgState->fModifiedInterceptCtrl = false;
5583	pDbgState->fModifiedXcptBitmap = false;
5584	pDbgState->fModifiedInterceptRdCRx = false;
5585	pDbgState->fModifiedInterceptWrCRx = false;
5586	pDbgState->fModifiedInterceptRdDRx = false;
5587	pDbgState->fModifiedInterceptWrDRx = false;
5588
5589	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
5590	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
5591
5592	/* We don't really need to zero these. */
5593	pDbgState->bmInterceptExtra = 0;
5594	pDbgState->bmXcptExtra = 0;
5595	pDbgState->bmInterceptRdCRxExtra = 0;
5596	pDbgState->bmInterceptWrCRxExtra = 0;
5597	pDbgState->bmInterceptRdDRxExtra = 0;
5598	pDbgState->bmInterceptWrDRxExtra = 0;
5599	}
5600
5601
5602	/**
5603	* Updates the VMCB fields with changes requested by @a pDbgState.
5604	*
5605	* This is performed after hmR0SvmPreRunGuestDebugStateUpdate as well
5606	* immediately before executing guest code, i.e. when interrupts are disabled.
5607	* We don't check status codes here as we cannot easily assert or return in the
5608	* latter case.
5609	*
5610	* @param pSvmTransient The SVM-transient structure.
5611	* @param pDbgState The debug state.
5612	*/
5613	static void hmR0SvmPreRunGuestDebugStateApply(PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5614	{
5615	/*
5616	* Ensure desired flags in VMCS control fields are set.
5617	*/
5618	PSVMVMCB const pVmcb = pSvmTransient->pVmcb;
5619	#define ADD_EXTRA_INTERCEPTS(a_VmcbCtrlField, a_bmExtra, a_fModified) do { \
5620	if ((pVmcb->ctrl. a_VmcbCtrlField & (a_bmExtra)) != (a_bmExtra)) \
5621	{ \
5622	pVmcb->ctrl. a_VmcbCtrlField \|= (a_bmExtra); \
5623	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS; \
5624	Log6Func((#a_VmcbCtrlField ": %#RX64\n", pVmcb->ctrl. a_VmcbCtrlField)); \
5625	(a_fModified) = true; \
5626	} \
5627	} while (0)
5628	ADD_EXTRA_INTERCEPTS(u64InterceptCtrl, pDbgState->bmInterceptExtra, pDbgState->fModifiedInterceptCtrl);
5629	ADD_EXTRA_INTERCEPTS(u32InterceptXcpt, pDbgState->bmXcptExtra, pDbgState->fModifiedXcptBitmap);
5630	ADD_EXTRA_INTERCEPTS(u16InterceptRdCRx, pDbgState->bmInterceptRdCRxExtra, pDbgState->fModifiedInterceptRdCRx);
5631	ADD_EXTRA_INTERCEPTS(u16InterceptWrCRx, pDbgState->bmInterceptWrCRxExtra, pDbgState->fModifiedInterceptWrCRx);
5632	ADD_EXTRA_INTERCEPTS(u16InterceptRdDRx, pDbgState->bmInterceptRdDRxExtra, pDbgState->fModifiedInterceptRdDRx);
5633	ADD_EXTRA_INTERCEPTS(u16InterceptWrDRx, pDbgState->bmInterceptWrDRxExtra, pDbgState->fModifiedInterceptWrDRx);
5634	#undef ADD_EXTRA_INTERCEPTS
5635	}
5636
5637
5638	/**
5639	* Restores VMCB fields that were changed by hmR0SvmPreRunGuestDebugStateApply
5640	* for re-entry next time around.
5641	*
5642	* @returns Strict VBox status code (i.e. informational status codes too).
5643	* @param pSvmTransient The SVM-transient structure.
5644	* @param pDbgState The debug state.
5645	*/
5646	static void hmR0SvmRunDebugStateRevert(PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5647	{
5648	/*
5649	* Restore VM-exit control settings as we may not reenter this function the
5650	* next time around.
5651	*/
5652	PSVMVMCB const pVmcb = pSvmTransient->pVmcb;
5653
5654	#define RESTORE_INTERCEPTS(a_VmcbCtrlField, a_bmInitial, a_fModified) do { \
5655	if ((a_fModified)) \
5656	{ \
5657	pVmcb->ctrl. a_VmcbCtrlField = (a_bmInitial); \
5658	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS; \
5659	} \
5660	} while (0)
5661	RESTORE_INTERCEPTS(u64InterceptCtrl, pDbgState->bmInterceptInitial, pDbgState->fModifiedInterceptCtrl);
5662	RESTORE_INTERCEPTS(u32InterceptXcpt, pDbgState->bmXcptInitial, pDbgState->fModifiedXcptBitmap);
5663	RESTORE_INTERCEPTS(u16InterceptRdCRx, pDbgState->bmInterceptRdCRxInitial, pDbgState->fModifiedInterceptRdCRx);
5664	RESTORE_INTERCEPTS(u16InterceptWrCRx, pDbgState->bmInterceptWrCRxInitial, pDbgState->fModifiedInterceptWrCRx);
5665	RESTORE_INTERCEPTS(u16InterceptRdDRx, pDbgState->bmInterceptRdDRxInitial, pDbgState->fModifiedInterceptRdDRx);
5666	RESTORE_INTERCEPTS(u16InterceptWrDRx, pDbgState->bmInterceptWrDRxInitial, pDbgState->fModifiedInterceptWrDRx);
5667	#undef RESTORE_INTERCEPTS
5668	}
5669
5670
5671	/**
5672	* Configures VM-exit controls for current DBGF and DTrace settings.
5673	*
5674	* This updates @a pDbgState and the VMCB execution control fields (in the debug
5675	* state) to reflect the necessary VM-exits demanded by DBGF and DTrace.
5676	*
5677	* @param pVCpu The cross context virtual CPU structure.
5678	* @param pSvmTransient The SVM-transient structure. May update
5679	* fUpdatedTscOffsettingAndPreemptTimer.
5680	* @param pDbgState The debug state.
5681	*/
5682	static void hmR0SvmPreRunGuestDebugStateUpdate(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5683	{
5684	/*
5685	* Take down the dtrace serial number so we can spot changes.
5686	*/
5687	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
5688	ASMCompilerBarrier();
5689
5690	/*
5691	* Clear data members that we'll be rebuilding here.
5692	*/
5693	pDbgState->bmXcptExtra = 0;
5694	pDbgState->bmInterceptExtra = 0;
5695	pDbgState->bmInterceptRdCRxExtra = 0;
5696	pDbgState->bmInterceptWrCRxExtra = 0;
5697	pDbgState->bmInterceptRdDRxExtra = 0;
5698	pDbgState->bmInterceptWrDRxExtra = 0;
5699	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
5700	pDbgState->bmExitsToCheck[i] = 0;
5701
5702	/*
5703	* Software interrupts (INT XXh)
5704	*/
5705	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5706	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
5707	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
5708	{
5709	pDbgState->bmInterceptExtra \|= SVM_CTRL_INTERCEPT_INTN;
5710	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_SWINT);
5711	}
5712
5713	/*
5714	* INT3 breakpoints - triggered by #BP exceptions.
5715	*/
5716	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
5717	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
5718
5719	/*
5720	* Exception bitmap and XCPT events+probes.
5721	*/
5722	#define SET_XCPT(a_iXcpt) do { \
5723	pDbgState->bmXcptExtra \|= RT_BIT_32(a_iXcpt); \
5724	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_XCPT_0 + (a_iXcpt)); \
5725	} while (0)
5726
5727	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
5728	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
5729	SET_XCPT(iXcpt);
5730
5731	if (VBOXVMM_XCPT_DE_ENABLED()) SET_XCPT(X86_XCPT_DE);
5732	if (VBOXVMM_XCPT_DB_ENABLED()) SET_XCPT(X86_XCPT_DB);
5733	if (VBOXVMM_XCPT_BP_ENABLED()) SET_XCPT(X86_XCPT_BP);
5734	if (VBOXVMM_XCPT_OF_ENABLED()) SET_XCPT(X86_XCPT_OF);
5735	if (VBOXVMM_XCPT_BR_ENABLED()) SET_XCPT(X86_XCPT_BR);
5736	if (VBOXVMM_XCPT_UD_ENABLED()) SET_XCPT(X86_XCPT_UD);
5737	if (VBOXVMM_XCPT_NM_ENABLED()) SET_XCPT(X86_XCPT_NM);
5738	if (VBOXVMM_XCPT_DF_ENABLED()) SET_XCPT(X86_XCPT_DF);
5739	if (VBOXVMM_XCPT_TS_ENABLED()) SET_XCPT(X86_XCPT_TS);
5740	if (VBOXVMM_XCPT_NP_ENABLED()) SET_XCPT(X86_XCPT_NP);
5741	if (VBOXVMM_XCPT_SS_ENABLED()) SET_XCPT(X86_XCPT_SS);
5742	if (VBOXVMM_XCPT_GP_ENABLED()) SET_XCPT(X86_XCPT_GP);
5743	if (VBOXVMM_XCPT_PF_ENABLED()) SET_XCPT(X86_XCPT_PF);
5744	if (VBOXVMM_XCPT_MF_ENABLED()) SET_XCPT(X86_XCPT_MF);
5745	if (VBOXVMM_XCPT_AC_ENABLED()) SET_XCPT(X86_XCPT_AC);
5746	if (VBOXVMM_XCPT_XF_ENABLED()) SET_XCPT(X86_XCPT_XF);
5747	if (VBOXVMM_XCPT_VE_ENABLED()) SET_XCPT(X86_XCPT_VE);
5748	if (VBOXVMM_XCPT_SX_ENABLED()) SET_XCPT(X86_XCPT_SX);
5749
5750	#undef SET_XCPT
5751
5752	/*
5753	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
5754	*
5755	* Note! This is the reverse of what hmR0SvmHandleExitDtraceEvents does.
5756	* So, when adding/changing/removing please don't forget to update it.
5757	*
5758	* Some of the macros are picking up local variables to save horizontal space,
5759	* (being able to see it in a table is the lesser evil here).
5760	*/
5761	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
5762	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
5763	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
5764	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
5765	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
5766	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
5767	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
5768	} else do { } while (0)
5769	#define SET_INCP_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fInterceptCtrl) \
5770	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
5771	{ \
5772	(pDbgState)->bmInterceptExtra \|= (a_fInterceptCtrl); \
5773	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
5774	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
5775	} else do { } while (0)
5776
5777	/** @todo double check these */
5778	/** @todo Check what more AMD-V specific we can intercept. */
5779	//SET_INCP_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, SVM_EXIT_TASK_SWITCH, SVM_CTRL_INTERCEPT_TASK_SWITCH);
5780	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, SVM_EXIT_TASK_SWITCH);
5781	SET_INCP_XBM_IF_EITHER_EN(INSTR_VMM_CALL, SVM_EXIT_VMMCALL, SVM_CTRL_INTERCEPT_VMMCALL);
5782	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, SVM_EXIT_VMMCALL);
5783	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_VMRUN, SVM_EXIT_VMRUN, SVM_CTRL_INTERCEPT_VMRUN);
5784	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_VMRUN, SVM_EXIT_VMRUN);
5785	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_VMLOAD, SVM_EXIT_VMLOAD, SVM_CTRL_INTERCEPT_VMLOAD);
5786	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_VMLOAD, SVM_EXIT_VMLOAD);
5787	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_VMSAVE, SVM_EXIT_VMSAVE, SVM_CTRL_INTERCEPT_VMSAVE);
5788	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_VMSAVE, SVM_EXIT_VMSAVE);
5789	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_STGI, SVM_EXIT_STGI, SVM_CTRL_INTERCEPT_STGI);
5790	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_STGI, SVM_EXIT_STGI);
5791	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_CLGI, SVM_EXIT_CLGI, SVM_CTRL_INTERCEPT_CLGI);
5792	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_CLGI, SVM_EXIT_CLGI);
5793
5794	SET_INCP_XBM_IF_EITHER_EN(INSTR_CPUID, SVM_EXIT_CPUID, SVM_CTRL_INTERCEPT_CPUID);
5795	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, SVM_EXIT_CPUID);
5796	SET_INCP_XBM_IF_EITHER_EN(INSTR_HALT, SVM_EXIT_HLT, SVM_CTRL_INTERCEPT_HLT);
5797	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, SVM_EXIT_HLT);
5798	SET_INCP_XBM_IF_EITHER_EN(INSTR_INVD, SVM_EXIT_INVD, SVM_CTRL_INTERCEPT_INVD);
5799	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, SVM_EXIT_INVD);
5800	SET_INCP_XBM_IF_EITHER_EN(INSTR_INVLPG, SVM_EXIT_INVLPG, SVM_CTRL_INTERCEPT_INVLPG);
5801	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, SVM_EXIT_INVLPG);
5802	SET_INCP_XBM_IF_EITHER_EN(INSTR_RDPMC, SVM_EXIT_RDPMC, SVM_CTRL_INTERCEPT_RDPMC);
5803	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, SVM_EXIT_RDPMC);
5804	SET_INCP_XBM_IF_EITHER_EN(INSTR_RDTSC, SVM_EXIT_RDTSC, SVM_CTRL_INTERCEPT_RDTSC);
5805	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, SVM_EXIT_RDTSC);
5806	SET_INCP_XBM_IF_EITHER_EN(INSTR_RDTSCP, SVM_EXIT_RDTSCP, SVM_CTRL_INTERCEPT_RDTSCP);
5807	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, SVM_EXIT_RDTSCP);
5808	SET_INCP_XBM_IF_EITHER_EN(INSTR_RSM, SVM_EXIT_RSM, SVM_CTRL_INTERCEPT_RSM);
5809	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, SVM_EXIT_RSM);
5810
5811	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
5812	pDbgState->bmInterceptRdCRxExtra = 0xffff;
5813	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ) \|\| IS_EITHER_ENABLED(pVM, EXIT_CRX_READ))
5814	ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_READ_CR0, SVM_EXIT_READ_CR15 + 1);
5815
5816	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
5817	pDbgState->bmInterceptWrCRxExtra = 0xffff;
5818	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE) \|\| IS_EITHER_ENABLED(pVM, EXIT_CRX_WRITE))
5819	{
5820	ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_WRITE_CR0, SVM_EXIT_WRITE_CR15 + 1);
5821	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_CR0_SEL_WRITE);
5822	}
5823
5824	if (IS_EITHER_ENABLED(pVM, INSTR_DRX_READ))
5825	pDbgState->bmInterceptRdDRxExtra = 0xffff;
5826	if (IS_EITHER_ENABLED(pVM, INSTR_DRX_READ) \|\| IS_EITHER_ENABLED(pVM, EXIT_DRX_READ))
5827	ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_READ_DR0, SVM_EXIT_READ_DR15 + 1);
5828
5829	if (IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
5830	pDbgState->bmInterceptWrDRxExtra = 0xffff;
5831	if (IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE) \|\| IS_EITHER_ENABLED(pVM, EXIT_DRX_WRITE))
5832	ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_WRITE_DR0, SVM_EXIT_WRITE_DR15 + 1);
5833
5834	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RDMSR, SVM_EXIT_MSR); /** @todo modify bitmap to intercept almost everything? (Clearing MSR_PROT just means no intercepts.) */
5835	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, SVM_EXIT_MSR);
5836	SET_ONLY_XBM_IF_EITHER_EN(INSTR_WRMSR, SVM_EXIT_MSR); /** @todo ditto */
5837	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, SVM_EXIT_MSR);
5838	SET_INCP_XBM_IF_EITHER_EN(INSTR_MWAIT, SVM_EXIT_MWAIT, SVM_CTRL_INTERCEPT_MWAIT);
5839	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, SVM_EXIT_MWAIT);
5840	if (ASMBitTest(pDbgState->bmExitsToCheck, SVM_EXIT_MWAIT))
5841	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_MWAIT_ARMED);
5842	SET_INCP_XBM_IF_EITHER_EN(INSTR_MONITOR, SVM_EXIT_MONITOR, SVM_CTRL_INTERCEPT_MONITOR);
5843	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, SVM_EXIT_MONITOR);
5844	SET_INCP_XBM_IF_EITHER_EN(INSTR_PAUSE, SVM_EXIT_PAUSE, SVM_CTRL_INTERCEPT_PAUSE);
5845	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, SVM_EXIT_PAUSE);
5846	SET_INCP_XBM_IF_EITHER_EN(INSTR_SIDT, SVM_EXIT_IDTR_READ, SVM_CTRL_INTERCEPT_IDTR_READS);
5847	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, SVM_EXIT_IDTR_READ);
5848	SET_INCP_XBM_IF_EITHER_EN(INSTR_LIDT, SVM_EXIT_IDTR_WRITE, SVM_CTRL_INTERCEPT_IDTR_WRITES);
5849	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, SVM_EXIT_IDTR_WRITE);
5850	SET_INCP_XBM_IF_EITHER_EN(INSTR_SGDT, SVM_EXIT_GDTR_READ, SVM_CTRL_INTERCEPT_GDTR_READS);
5851	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, SVM_EXIT_GDTR_READ);
5852	SET_INCP_XBM_IF_EITHER_EN(INSTR_LGDT, SVM_EXIT_GDTR_WRITE, SVM_CTRL_INTERCEPT_GDTR_WRITES);
5853	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, SVM_EXIT_GDTR_WRITE);
5854	SET_INCP_XBM_IF_EITHER_EN(INSTR_SLDT, SVM_EXIT_LDTR_READ, SVM_CTRL_INTERCEPT_LDTR_READS);
5855	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, SVM_EXIT_LDTR_READ);
5856	SET_INCP_XBM_IF_EITHER_EN(INSTR_LLDT, SVM_EXIT_LDTR_WRITE, SVM_CTRL_INTERCEPT_LDTR_WRITES);
5857	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, SVM_EXIT_LDTR_WRITE);
5858	SET_INCP_XBM_IF_EITHER_EN(INSTR_STR, SVM_EXIT_TR_READ, SVM_CTRL_INTERCEPT_TR_READS);
5859	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, SVM_EXIT_TR_READ);
5860	SET_INCP_XBM_IF_EITHER_EN(INSTR_LTR, SVM_EXIT_TR_WRITE, SVM_CTRL_INTERCEPT_TR_WRITES);
5861	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, SVM_EXIT_TR_WRITE);
5862	SET_INCP_XBM_IF_EITHER_EN(INSTR_WBINVD, SVM_EXIT_WBINVD, SVM_CTRL_INTERCEPT_WBINVD);
5863	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, SVM_EXIT_WBINVD);
5864	SET_INCP_XBM_IF_EITHER_EN(INSTR_XSETBV, SVM_EXIT_XSETBV, SVM_CTRL_INTERCEPT_XSETBV);
5865	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, SVM_EXIT_XSETBV);
5866
5867	if (DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_TRIPLE_FAULT))
5868	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_SHUTDOWN);
5869
5870	#undef IS_EITHER_ENABLED
5871	#undef SET_ONLY_XBM_IF_EITHER_EN
5872	#undef SET_INCP_XBM_IF_EITHER_EN
5873
5874	/*
5875	* Sanitize the control stuff.
5876	*/
5877	/** @todo filter out unsupported stuff? */
5878	if ( pVCpu->hmr0.s.fDebugWantRdTscExit
5879	!= RT_BOOL(pDbgState->bmInterceptExtra & (SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP)))
5880	{
5881	pVCpu->hmr0.s.fDebugWantRdTscExit ^= true;
5882	/// @todo pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
5883	RT_NOREF(pSvmTransient);
5884	}
5885
5886	Log6(("HM: debug state: bmInterceptExtra=%#RX64 bmXcptExtra=%#RX32%s%s%s%s bmExitsToCheck=%08RX32'%08RX32'%08RX32'%08RX32'%08RX32\n",
5887	pDbgState->bmInterceptExtra, pDbgState->bmXcptExtra,
5888	pDbgState->bmInterceptRdCRxExtra ? " rd-cr" : "",
5889	pDbgState->bmInterceptWrCRxExtra ? " wr-cr" : "",
5890	pDbgState->bmInterceptRdDRxExtra ? " rd-dr" : "",
5891	pDbgState->bmInterceptWrDRxExtra ? " wr-dr" : "",
5892	pDbgState->bmExitsToCheck[0],
5893	pDbgState->bmExitsToCheck[1],
5894	pDbgState->bmExitsToCheck[2],
5895	pDbgState->bmExitsToCheck[3],
5896	pDbgState->bmExitsToCheck[4]));
5897	}
5898
5899
5900	/**
5901	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
5902	* appropriate.
5903	*
5904	* The caller has checked the VM-exit against the SVMRUNDBGSTATE::bmExitsToCheck
5905	* bitmap.
5906	*
5907	* @returns Strict VBox status code (i.e. informational status codes too).
5908	* @param pVCpu The cross context virtual CPU structure.
5909	* @param pSvmTransient The SVM-transient structure.
5910	* @param uExitCode The VM-exit code.
5911	*
5912	* @remarks The name of this function is displayed by dtrace, so keep it short
5913	* and to the point. No longer than 33 chars long, please.
5914	*/
5915	static VBOXSTRICTRC hmR0SvmHandleExitDtraceEvents(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, uint64_t uExitCode)
5916	{
5917	/*
5918	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
5919	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
5920	*
5921	* Note! This is the reverse operation of what hmR0SvmPreRunGuestDebugStateUpdate
5922	* does. Must add/change/remove both places. Same ordering, please.
5923	*
5924	* Added/removed events must also be reflected in the next section
5925	* where we dispatch dtrace events.
5926	*/
5927	bool fDtrace1 = false;
5928	bool fDtrace2 = false;
5929	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
5930	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
5931	uint64_t uEventArg = 0;
5932	#define SET_XCPT(a_XcptName) \
5933	do { \
5934	enmEvent2 = RT_CONCAT(DBGFEVENT_XCPT_, a_XcptName); \
5935	fDtrace2 = RT_CONCAT3(VBOXVMM_XCPT_, a_XcptName, _ENABLED)(); \
5936	} while (0)
5937	#define SET_EXIT(a_EventSubName) \
5938	do { \
5939	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
5940	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
5941	} while (0)
5942	#define SET_BOTH(a_EventSubName) \
5943	do { \
5944	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
5945	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
5946	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
5947	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
5948	} while (0)
5949	switch (uExitCode)
5950	{
5951	case SVM_EXIT_SWINT:
5952	enmEvent2 = DBGFEVENT_INTERRUPT_SOFTWARE;
5953	fDtrace2 = VBOXVMM_INT_SOFTWARE_ENABLED();
5954	uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1;
5955	break;
5956
5957	case SVM_EXIT_XCPT_DE: SET_XCPT(DE); break;
5958	case SVM_EXIT_XCPT_DB: SET_XCPT(DB); break;
5959	case SVM_EXIT_XCPT_BP: SET_XCPT(BP); break;
5960	case SVM_EXIT_XCPT_OF: SET_XCPT(OF); break;
5961	case SVM_EXIT_XCPT_BR: SET_XCPT(BR); break;
5962	case SVM_EXIT_XCPT_UD: SET_XCPT(UD); break;
5963	case SVM_EXIT_XCPT_NM: SET_XCPT(NM); break;
5964	case SVM_EXIT_XCPT_DF: SET_XCPT(DF); break;
5965	case SVM_EXIT_XCPT_TS: SET_XCPT(TS); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5966	case SVM_EXIT_XCPT_NP: SET_XCPT(NP); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5967	case SVM_EXIT_XCPT_SS: SET_XCPT(SS); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5968	case SVM_EXIT_XCPT_GP: SET_XCPT(GP); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5969	case SVM_EXIT_XCPT_PF: SET_XCPT(PF); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5970	case SVM_EXIT_XCPT_MF: SET_XCPT(MF); break;
5971	case SVM_EXIT_XCPT_AC: SET_XCPT(AC); break;
5972	case SVM_EXIT_XCPT_XF: SET_XCPT(XF); break;
5973	case SVM_EXIT_XCPT_VE: SET_XCPT(VE); break;
5974	case SVM_EXIT_XCPT_SX: SET_XCPT(SX); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5975
5976	case SVM_EXIT_XCPT_2: enmEvent2 = DBGFEVENT_XCPT_02; break;
5977	case SVM_EXIT_XCPT_9: enmEvent2 = DBGFEVENT_XCPT_09; break;
5978	case SVM_EXIT_XCPT_15: enmEvent2 = DBGFEVENT_XCPT_0f; break;
5979	case SVM_EXIT_XCPT_18: enmEvent2 = DBGFEVENT_XCPT_MC; break;
5980	case SVM_EXIT_XCPT_21: enmEvent2 = DBGFEVENT_XCPT_15; break;
5981	case SVM_EXIT_XCPT_22: enmEvent2 = DBGFEVENT_XCPT_16; break;
5982	case SVM_EXIT_XCPT_23: enmEvent2 = DBGFEVENT_XCPT_17; break;
5983	case SVM_EXIT_XCPT_24: enmEvent2 = DBGFEVENT_XCPT_18; break;
5984	case SVM_EXIT_XCPT_25: enmEvent2 = DBGFEVENT_XCPT_19; break;
5985	case SVM_EXIT_XCPT_26: enmEvent2 = DBGFEVENT_XCPT_1a; break;
5986	case SVM_EXIT_XCPT_27: enmEvent2 = DBGFEVENT_XCPT_1b; break;
5987	case SVM_EXIT_XCPT_28: enmEvent2 = DBGFEVENT_XCPT_1c; break;
5988	case SVM_EXIT_XCPT_29: enmEvent2 = DBGFEVENT_XCPT_1d; break;
5989	case SVM_EXIT_XCPT_31: enmEvent2 = DBGFEVENT_XCPT_1f; break;
5990
5991	case SVM_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
5992	case SVM_EXIT_VMMCALL: SET_BOTH(VMM_CALL); break;
5993	case SVM_EXIT_VMRUN: SET_BOTH(SVM_VMRUN); break;
5994	case SVM_EXIT_VMLOAD: SET_BOTH(SVM_VMLOAD); break;
5995	case SVM_EXIT_VMSAVE: SET_BOTH(SVM_VMSAVE); break;
5996	case SVM_EXIT_STGI: SET_BOTH(SVM_STGI); break;
5997	case SVM_EXIT_CLGI: SET_BOTH(SVM_CLGI); break;
5998	case SVM_EXIT_CPUID: SET_BOTH(CPUID); break;
5999	case SVM_EXIT_HLT: SET_BOTH(HALT); break;
6000	case SVM_EXIT_INVD: SET_BOTH(INVD); break;
6001	case SVM_EXIT_INVLPG: SET_BOTH(INVLPG); break;
6002	case SVM_EXIT_RDPMC: SET_BOTH(RDPMC); break;
6003	case SVM_EXIT_RDTSC: SET_BOTH(RDTSC); break;
6004	case SVM_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
6005	case SVM_EXIT_RSM: SET_BOTH(RSM); break;
6006
6007	case SVM_EXIT_READ_CR0: case SVM_EXIT_READ_CR1: case SVM_EXIT_READ_CR2: case SVM_EXIT_READ_CR3:
6008	case SVM_EXIT_READ_CR4: case SVM_EXIT_READ_CR5: case SVM_EXIT_READ_CR6: case SVM_EXIT_READ_CR7:
6009	case SVM_EXIT_READ_CR8: case SVM_EXIT_READ_CR9: case SVM_EXIT_READ_CR10: case SVM_EXIT_READ_CR11:
6010	case SVM_EXIT_READ_CR12: case SVM_EXIT_READ_CR13: case SVM_EXIT_READ_CR14: case SVM_EXIT_READ_CR15:
6011	SET_BOTH(CRX_READ);
6012	uEventArg = uExitCode - SVM_EXIT_READ_CR0;
6013	break;
6014	case SVM_EXIT_WRITE_CR0: case SVM_EXIT_WRITE_CR1: case SVM_EXIT_WRITE_CR2: case SVM_EXIT_WRITE_CR3:
6015	case SVM_EXIT_WRITE_CR4: case SVM_EXIT_WRITE_CR5: case SVM_EXIT_WRITE_CR6: case SVM_EXIT_WRITE_CR7:
6016	case SVM_EXIT_WRITE_CR8: case SVM_EXIT_WRITE_CR9: case SVM_EXIT_WRITE_CR10: case SVM_EXIT_WRITE_CR11:
6017	case SVM_EXIT_WRITE_CR12: case SVM_EXIT_WRITE_CR13: case SVM_EXIT_WRITE_CR14: case SVM_EXIT_WRITE_CR15:
6018	case SVM_EXIT_CR0_SEL_WRITE:
6019	SET_BOTH(CRX_WRITE);
6020	uEventArg = uExitCode - SVM_EXIT_WRITE_CR0;
6021	break;
6022	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
6023	case SVM_EXIT_READ_DR4: case SVM_EXIT_READ_DR5: case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7:
6024	case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9: case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11:
6025	case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13: case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
6026	SET_BOTH(DRX_READ);
6027	uEventArg = uExitCode - SVM_EXIT_READ_DR0;
6028	break;
6029	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
6030	case SVM_EXIT_WRITE_DR4: case SVM_EXIT_WRITE_DR5: case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7:
6031	case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9: case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11:
6032	case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13: case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
6033	SET_BOTH(DRX_WRITE);
6034	uEventArg = uExitCode - SVM_EXIT_WRITE_DR0;
6035	break;
6036	case SVM_EXIT_MSR:
6037	if (pSvmTransient->pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_WRITE)
6038	SET_BOTH(WRMSR);
6039	else
6040	SET_BOTH(RDMSR);
6041	break;
6042	case SVM_EXIT_MWAIT_ARMED:
6043	case SVM_EXIT_MWAIT: SET_BOTH(MWAIT); break;
6044	case SVM_EXIT_MONITOR: SET_BOTH(MONITOR); break;
6045	case SVM_EXIT_PAUSE: SET_BOTH(PAUSE); break;
6046	case SVM_EXIT_IDTR_READ: SET_BOTH(SIDT); break;
6047	case SVM_EXIT_IDTR_WRITE: SET_BOTH(LIDT); break;
6048	case SVM_EXIT_GDTR_READ: SET_BOTH(SGDT); break;
6049	case SVM_EXIT_GDTR_WRITE: SET_BOTH(LGDT); break;
6050	case SVM_EXIT_LDTR_READ: SET_BOTH(SLDT); break;
6051	case SVM_EXIT_LDTR_WRITE: SET_BOTH(LLDT); break;
6052	case SVM_EXIT_TR_READ: SET_BOTH(STR); break;
6053	case SVM_EXIT_TR_WRITE: SET_BOTH(LTR); break;
6054	case SVM_EXIT_WBINVD: SET_BOTH(WBINVD); break;
6055	case SVM_EXIT_XSETBV: SET_BOTH(XSETBV); break;
6056
6057	case SVM_EXIT_SHUTDOWN:
6058	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
6059	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
6060	break;
6061
6062	default:
6063	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitCode));
6064	break;
6065	}
6066	#undef SET_BOTH
6067	#undef SET_EXIT
6068
6069	/*
6070	* Dtrace tracepoints go first. We do them here at once so we don't
6071	* have to copy the guest state saving and stuff a few dozen times.
6072	* Down side is that we've got to repeat the switch, though this time
6073	* we use enmEvent since the probes are a subset of what DBGF does.
6074	*/
6075	if (fDtrace1 \|\| fDtrace2)
6076	{
6077	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
6078	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6079	switch (enmEvent1)
6080	{
6081	/** @todo consider which extra parameters would be helpful for each probe. */
6082	case DBGFEVENT_END: break;
6083	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6084	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
6085	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
6086	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
6087	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
6088	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
6089	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
6090	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
6091	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
6092	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, (uint32_t)uEventArg); break;
6093	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, (uint32_t)uEventArg); break;
6094	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, (uint32_t)uEventArg); break;
6095	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, (uint32_t)uEventArg); break;
6096	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, (uint32_t)uEventArg, pCtx->cr2); break;
6097	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
6098	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
6099	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
6100	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
6101	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, (uint32_t)uEventArg); break;
6102	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
6103	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
6104	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
6105	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
6106	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
6107	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
6108	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
6109	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6110	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6111	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6112	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6113	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
6114	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
6115	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
6116	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
6117	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
6118	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
6119	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
6120	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
6121	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
6122	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
6123	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
6124	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
6125	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
6126	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
6127	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
6128	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
6129	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
6130	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
6131	case DBGFEVENT_INSTR_SVM_VMRUN: VBOXVMM_INSTR_SVM_VMRUN(pVCpu, pCtx); break;
6132	case DBGFEVENT_INSTR_SVM_VMLOAD: VBOXVMM_INSTR_SVM_VMLOAD(pVCpu, pCtx); break;
6133	case DBGFEVENT_INSTR_SVM_VMSAVE: VBOXVMM_INSTR_SVM_VMSAVE(pVCpu, pCtx); break;
6134	case DBGFEVENT_INSTR_SVM_STGI: VBOXVMM_INSTR_SVM_STGI(pVCpu, pCtx); break;
6135	case DBGFEVENT_INSTR_SVM_CLGI: VBOXVMM_INSTR_SVM_CLGI(pVCpu, pCtx); break;
6136	default: AssertMsgFailed(("enmEvent1=%d uExitCode=%d\n", enmEvent1, uExitCode)); break;
6137	}
6138	switch (enmEvent2)
6139	{
6140	/** @todo consider which extra parameters would be helpful for each probe. */
6141	case DBGFEVENT_END: break;
6142	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
6143	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
6144	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
6145	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
6146	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
6147	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
6148	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
6149	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
6150	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6151	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6152	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6153	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6154	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
6155	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
6156	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
6157	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
6158	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
6159	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
6160	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
6161	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
6162	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
6163	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
6164	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
6165	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
6166	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
6167	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
6168	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
6169	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
6170	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
6171	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
6172	case DBGFEVENT_EXIT_SVM_VMRUN: VBOXVMM_EXIT_SVM_VMRUN(pVCpu, pCtx); break;
6173	case DBGFEVENT_EXIT_SVM_VMLOAD: VBOXVMM_EXIT_SVM_VMLOAD(pVCpu, pCtx); break;
6174	case DBGFEVENT_EXIT_SVM_VMSAVE: VBOXVMM_EXIT_SVM_VMSAVE(pVCpu, pCtx); break;
6175	case DBGFEVENT_EXIT_SVM_STGI: VBOXVMM_EXIT_SVM_STGI(pVCpu, pCtx); break;
6176	case DBGFEVENT_EXIT_SVM_CLGI: VBOXVMM_EXIT_SVM_CLGI(pVCpu, pCtx); break;
6177	default: AssertMsgFailed(("enmEvent2=%d uExitCode=%d\n", enmEvent2, uExitCode)); break;
6178	}
6179	}
6180
6181	/*
6182	* Fire of the DBGF event, if enabled (our check here is just a quick one,
6183	* the DBGF call will do a full check).
6184	*
6185	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
6186	* Note! If we have to events, we prioritize the first, i.e. the instruction
6187	* one, in order to avoid event nesting.
6188	*/
6189	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6190	VBOXSTRICTRC rcStrict;
6191	if ( enmEvent1 != DBGFEVENT_END
6192	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
6193	{
6194	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6195	rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
6196	}
6197	else if ( enmEvent2 != DBGFEVENT_END
6198	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
6199	{
6200	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6201	rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
6202	}
6203	else
6204	rcStrict = VINF_SUCCESS;
6205	return rcStrict;
6206	}
6207
6208
6209	/**
6210	* Handles a guest \#VMEXIT (for all EXITCODE values except SVM_EXIT_INVALID),
6211	* debug variant.
6212	*
6213	* @returns Strict VBox status code (informational status codes included).
6214	* @param pVCpu The cross context virtual CPU structure.
6215	* @param pSvmTransient Pointer to the SVM transient structure.
6216	* @param pDbgState The runtime debug state.
6217	*/
6218	static VBOXSTRICTRC hmR0SvmDebugHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
6219	{
6220	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
6221	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
6222
6223	/*
6224	* Expensive (saves context) generic dtrace VM-exit probe.
6225	*/
6226	uint64_t const uExitCode = pSvmTransient->u64ExitCode;
6227	if (!VBOXVMM_R0_HMSVM_VMEXIT_ENABLED())
6228	{ /* more likely */ }
6229	else
6230	{
6231	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
6232	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, uExitCode, pSvmTransient->pVmcb);
6233	}
6234
6235	/*
6236	* Check for single stepping event if we're stepping.
6237	*/
6238	if (pVCpu->hm.s.fSingleInstruction)
6239	{
6240	switch (uExitCode)
6241	{
6242	/* Various events: */
6243	case SVM_EXIT_XCPT_0: case SVM_EXIT_XCPT_1: case SVM_EXIT_XCPT_2: case SVM_EXIT_XCPT_3:
6244	case SVM_EXIT_XCPT_4: case SVM_EXIT_XCPT_5: case SVM_EXIT_XCPT_6: case SVM_EXIT_XCPT_7:
6245	case SVM_EXIT_XCPT_8: case SVM_EXIT_XCPT_9: case SVM_EXIT_XCPT_10: case SVM_EXIT_XCPT_11:
6246	case SVM_EXIT_XCPT_12: case SVM_EXIT_XCPT_13: case SVM_EXIT_XCPT_14: case SVM_EXIT_XCPT_15:
6247	case SVM_EXIT_XCPT_16: case SVM_EXIT_XCPT_17: case SVM_EXIT_XCPT_18: case SVM_EXIT_XCPT_19:
6248	case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
6249	case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
6250	case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
6251	case SVM_EXIT_INTR:
6252	case SVM_EXIT_NMI:
6253	case SVM_EXIT_VINTR:
6254	case SVM_EXIT_NPF:
6255	case SVM_EXIT_AVIC_NOACCEL:
6256
6257	/* Instruction specific VM-exits: */
6258	case SVM_EXIT_READ_CR0: case SVM_EXIT_READ_CR1: case SVM_EXIT_READ_CR2: case SVM_EXIT_READ_CR3:
6259	case SVM_EXIT_READ_CR4: case SVM_EXIT_READ_CR5: case SVM_EXIT_READ_CR6: case SVM_EXIT_READ_CR7:
6260	case SVM_EXIT_READ_CR8: case SVM_EXIT_READ_CR9: case SVM_EXIT_READ_CR10: case SVM_EXIT_READ_CR11:
6261	case SVM_EXIT_READ_CR12: case SVM_EXIT_READ_CR13: case SVM_EXIT_READ_CR14: case SVM_EXIT_READ_CR15:
6262	case SVM_EXIT_WRITE_CR0: case SVM_EXIT_WRITE_CR1: case SVM_EXIT_WRITE_CR2: case SVM_EXIT_WRITE_CR3:
6263	case SVM_EXIT_WRITE_CR4: case SVM_EXIT_WRITE_CR5: case SVM_EXIT_WRITE_CR6: case SVM_EXIT_WRITE_CR7:
6264	case SVM_EXIT_WRITE_CR8: case SVM_EXIT_WRITE_CR9: case SVM_EXIT_WRITE_CR10: case SVM_EXIT_WRITE_CR11:
6265	case SVM_EXIT_WRITE_CR12: case SVM_EXIT_WRITE_CR13: case SVM_EXIT_WRITE_CR14: case SVM_EXIT_WRITE_CR15:
6266	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
6267	case SVM_EXIT_READ_DR4: case SVM_EXIT_READ_DR5: case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7:
6268	case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9: case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11:
6269	case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13: case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
6270	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
6271	case SVM_EXIT_WRITE_DR4: case SVM_EXIT_WRITE_DR5: case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7:
6272	case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9: case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11:
6273	case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13: case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
6274	case SVM_EXIT_CR0_SEL_WRITE:
6275	case SVM_EXIT_IDTR_READ:
6276	case SVM_EXIT_GDTR_READ:
6277	case SVM_EXIT_LDTR_READ:
6278	case SVM_EXIT_TR_READ:
6279	case SVM_EXIT_IDTR_WRITE:
6280	case SVM_EXIT_GDTR_WRITE:
6281	case SVM_EXIT_LDTR_WRITE:
6282	case SVM_EXIT_TR_WRITE:
6283	case SVM_EXIT_RDTSC:
6284	case SVM_EXIT_RDPMC:
6285	case SVM_EXIT_PUSHF:
6286	case SVM_EXIT_POPF:
6287	case SVM_EXIT_CPUID:
6288	case SVM_EXIT_RSM:
6289	case SVM_EXIT_IRET:
6290	case SVM_EXIT_SWINT:
6291	case SVM_EXIT_INVD:
6292	case SVM_EXIT_PAUSE:
6293	case SVM_EXIT_HLT:
6294	case SVM_EXIT_INVLPG:
6295	case SVM_EXIT_INVLPGA:
6296	case SVM_EXIT_IOIO:
6297	case SVM_EXIT_MSR:
6298	case SVM_EXIT_TASK_SWITCH:
6299	case SVM_EXIT_VMRUN:
6300	case SVM_EXIT_VMMCALL:
6301	case SVM_EXIT_VMLOAD:
6302	case SVM_EXIT_VMSAVE:
6303	case SVM_EXIT_STGI:
6304	case SVM_EXIT_CLGI:
6305	case SVM_EXIT_SKINIT:
6306	case SVM_EXIT_RDTSCP:
6307	case SVM_EXIT_ICEBP:
6308	case SVM_EXIT_WBINVD:
6309	case SVM_EXIT_MONITOR:
6310	case SVM_EXIT_MWAIT:
6311	case SVM_EXIT_MWAIT_ARMED:
6312	case SVM_EXIT_XSETBV:
6313	case SVM_EXIT_RDPRU:
6314	case SVM_EXIT_WRITE_EFER_TRAP:
6315	case SVM_EXIT_WRITE_CR0_TRAP: case SVM_EXIT_WRITE_CR1_TRAP: case SVM_EXIT_WRITE_CR2_TRAP: case SVM_EXIT_WRITE_CR3_TRAP:
6316	case SVM_EXIT_WRITE_CR4_TRAP: case SVM_EXIT_WRITE_CR5_TRAP: case SVM_EXIT_WRITE_CR6_TRAP: case SVM_EXIT_WRITE_CR7_TRAP:
6317	case SVM_EXIT_WRITE_CR8_TRAP: case SVM_EXIT_WRITE_CR9_TRAP: case SVM_EXIT_WRITE_CR10_TRAP: case SVM_EXIT_WRITE_CR11_TRAP:
6318	case SVM_EXIT_WRITE_CR12_TRAP: case SVM_EXIT_WRITE_CR13_TRAP: case SVM_EXIT_WRITE_CR14_TRAP: case SVM_EXIT_WRITE_CR15_TRAP:
6319	case SVM_EXIT_MCOMMIT:
6320	{
6321	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6322	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
6323	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
6324	{
6325	Log6Func(("VINF_EM_DBG_STEPPED: %04x:%08RX64 (exit %u)\n",
6326	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uExitCode));
6327	return VINF_EM_DBG_STEPPED;
6328	}
6329	break;
6330	}
6331
6332	/* Errors and unexpected events: */
6333	case SVM_EXIT_FERR_FREEZE:
6334	case SVM_EXIT_SHUTDOWN:
6335	case SVM_EXIT_AVIC_INCOMPLETE_IPI:
6336	break;
6337
6338	case SVM_EXIT_SMI:
6339	case SVM_EXIT_INIT:
6340	default:
6341	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitCode));
6342	break;
6343	}
6344	}
6345
6346	/*
6347	* Check for debugger event breakpoints and dtrace probes.
6348	*/
6349	if ( uExitCode < sizeof(pDbgState->bmExitsToCheck) * 8U
6350	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitCode) )
6351	{
6352	VBOXSTRICTRC rcStrict = hmR0SvmHandleExitDtraceEvents(pVCpu, pSvmTransient, uExitCode);
6353	if (rcStrict != VINF_SUCCESS)
6354	{
6355	Log6Func(("%04x:%08RX64 (exit %u) -> %Rrc\n",
6356	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uExitCode, VBOXSTRICTRC_VAL(rcStrict) ));
6357	return rcStrict;
6358	}
6359	}
6360
6361	/*
6362	* Normal processing.
6363	*/
6364	return hmR0SvmHandleExit(pVCpu, pSvmTransient);
6365	}
6366
6367
6368	/**
6369	* Runs the guest code using AMD-V in single step mode.
6370	*
6371	* @returns Strict VBox status code.
6372	* @param pVCpu The cross context virtual CPU structure.
6373	* @param pcLoops Pointer to the number of executed loops.
6374	*/
6375	static VBOXSTRICTRC hmR0SvmRunGuestCodeDebug(PVMCPUCC pVCpu, uint32_t *pcLoops)
6376	{
6377	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops;
6378	Assert(pcLoops);
6379	Assert(*pcLoops <= cMaxResumeLoops);
6380
6381	SVMTRANSIENT SvmTransient;
6382	RT_ZERO(SvmTransient);
6383	SvmTransient.fUpdateTscOffsetting = true;
6384	SvmTransient.pVmcb = pVCpu->hmr0.s.svm.pVmcb;
6385
6386	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6387
6388	/* Set HMCPU indicators. */
6389	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
6390	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
6391	pVCpu->hmr0.s.fDebugWantRdTscExit = false;
6392	pVCpu->hmr0.s.fUsingDebugLoop = true;
6393
6394	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
6395	SVMRUNDBGSTATE DbgState;
6396	hmR0SvmRunDebugStateInit(pVCpu, &SvmTransient, &DbgState);
6397	hmR0SvmPreRunGuestDebugStateUpdate(pVCpu, &SvmTransient, &DbgState);
6398
6399	/*
6400	* The loop.
6401	*/
6402	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_5;
6403	for (;;)
6404	{
6405	Assert(!HMR0SuspendPending());
6406	AssertMsg(pVCpu->hmr0.s.idEnteredCpu == RTMpCpuId(),
6407	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hmr0.s.idEnteredCpu,
6408	(unsigned)RTMpCpuId(), *pcLoops));
6409	bool fStepping = pVCpu->hm.s.fSingleInstruction;
6410
6411	/* Set up VM-execution controls the next two can respond to. */
6412	hmR0SvmPreRunGuestDebugStateApply(&SvmTransient, &DbgState);
6413
6414	/* Preparatory work for running nested-guest code, this may force us to return to
6415	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
6416	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6417	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
6418	if (rc != VINF_SUCCESS)
6419	break;
6420
6421	/*
6422	* No longjmps to ring-3 from this point on!!!
6423	*
6424	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
6425	* better than a kernel panic. This also disables flushing of the R0-logger instance.
6426	*/
6427	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
6428
6429	/* Override any obnoxious code in the above two calls. */
6430	hmR0SvmPreRunGuestDebugStateApply(&SvmTransient, &DbgState);
6431	#if 0
6432	Log(("%04x:%08RX64 ds=%04x %04x:%08RX64 i=%#RX64\n",
6433	SvmTransient.pVmcb->guest.CS.u16Sel, SvmTransient.pVmcb->guest.u64RIP, SvmTransient.pVmcb->guest.DS.u16Sel,
6434	SvmTransient.pVmcb->guest.SS.u16Sel, SvmTransient.pVmcb->guest.u64RSP, SvmTransient.pVmcb->ctrl.EventInject.u));
6435	#endif
6436
6437	/*
6438	* Finally execute guest code.
6439	*/
6440	rc = hmR0SvmRunGuest(pVCpu, pVCpu->hmr0.s.svm.HCPhysVmcb);
6441
6442	/* Restore any residual host-state and save any bits shared between host and guest
6443	into the guest-CPU state. Re-enables interrupts! */
6444	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
6445	#if 0
6446	Log(("%04x:%08RX64 ds=%04x %04x:%08RX64 i=%#RX64 exit=%d\n",
6447	SvmTransient.pVmcb->guest.CS.u16Sel, SvmTransient.pVmcb->guest.u64RIP, SvmTransient.pVmcb->guest.DS.u16Sel,
6448	SvmTransient.pVmcb->guest.SS.u16Sel, SvmTransient.pVmcb->guest.u64RSP, SvmTransient.pVmcb->ctrl.EventInject.u, SvmTransient.u64ExitCode));
6449	#endif
6450
6451	if (RT_LIKELY( rc == VINF_SUCCESS /* Check for VMRUN errors. */
6452	&& SvmTransient.u64ExitCode != SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
6453	{ /* very likely*/ }
6454	else
6455	{
6456	if (rc == VINF_SUCCESS)
6457	rc = VERR_SVM_INVALID_GUEST_STATE;
6458	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
6459	hmR0SvmReportWorldSwitchError(pVCpu, VBOXSTRICTRC_VAL(rc));
6460	return rc;
6461	}
6462
6463	/* Handle the #VMEXIT. */
6464	HMSVM_DEBUG_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
6465	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
6466	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, pVCpu->hmr0.s.svm.pVmcb);
6467	rc = hmR0SvmDebugHandleExit(pVCpu, &SvmTransient, &DbgState);
6468	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
6469	if (rc != VINF_SUCCESS)
6470	break;
6471	if (++(*pcLoops) >= cMaxResumeLoops)
6472	{
6473	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
6474	rc = VINF_EM_RAW_INTERRUPT;
6475	break;
6476	}
6477
6478	/*
6479	* Stepping: Did the RIP change, if so, consider it a single step.
6480	* Otherwise, make sure one of the TFs gets set.
6481	*/
6482	if (fStepping)
6483	{
6484	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6485	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
6486	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
6487	{
6488	Log6Func(("VINF_EM_DBG_STEPPED: %04x:%08RX64 (exit %u)\n",
6489	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, SvmTransient.u64ExitCode));
6490	rc = VINF_EM_DBG_STEPPED;
6491	break;
6492	}
6493	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
6494	}
6495
6496	/*
6497	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
6498	* Revert the state changes afterware so we can drop intercepts no longer needed.
6499	*/
6500	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
6501	{
6502	hmR0SvmPreRunGuestDebugStateUpdate(pVCpu, &SvmTransient, &DbgState);
6503	hmR0SvmRunDebugStateRevert(&SvmTransient, &DbgState);
6504	}
6505	}
6506
6507	/*
6508	* Clear the X86_EFL_TF if necessary.
6509	*/
6510	if (pVCpu->hmr0.s.fClearTrapFlag)
6511	{
6512	pVCpu->hmr0.s.fClearTrapFlag = false;
6513	pCtx->eflags.Bits.u1TF = 0;
6514	}
6515
6516	/* Restore HMCPU indicators. */
6517	pVCpu->hmr0.s.fUsingDebugLoop = false;
6518	pVCpu->hmr0.s.fDebugWantRdTscExit = false;
6519	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
6520
6521	/* Restore all controls applied by hmR0SvmPreRunGuestDebugStateApply above. */
6522	hmR0SvmRunDebugStateRevert(&SvmTransient, &DbgState);
6523
6524	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6525	return rc;
6526	}
6527
6528	/** @} */
6529
6530	#undef VMEXIT_CALL_RET
6531
6532
6533	#ifdef VBOX_STRICT
6534	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6535	# define HMSVM_ASSERT_PREEMPT_CPUID_VAR() \
6536	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6537
6538	# define HMSVM_ASSERT_PREEMPT_CPUID() \
6539	do \
6540	{ \
6541	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6542	AssertMsg(idAssertCpu == idAssertCpuNow, ("SVM %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6543	} while (0)
6544
6545	# define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pSvmTransient) \
6546	do { \
6547	AssertPtr((a_pVCpu)); \
6548	AssertPtr((a_pSvmTransient)); \
6549	Assert(ASMIntAreEnabled()); \
6550	HMSVM_ASSERT_PREEMPT_SAFE((a_pVCpu)); \
6551	HMSVM_ASSERT_PREEMPT_CPUID_VAR(); \
6552	Log4Func(("vcpu[%u] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (a_pVCpu)->idCpu)); \
6553	HMSVM_ASSERT_PREEMPT_SAFE((a_pVCpu)); \
6554	if (!VMMRZCallRing3IsEnabled((a_pVCpu))) \
6555	HMSVM_ASSERT_PREEMPT_CPUID(); \
6556	} while (0)
6557	#else
6558	# define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pSvmTransient) \
6559	do { \
6560	RT_NOREF2(a_pVCpu, a_pSvmTransient); \
6561	} while (0)
6562	#endif
6563
6564
6565	/**
6566	* Gets the IEM exception flags for the specified SVM event.
6567	*
6568	* @returns The IEM exception flags.
6569	* @param pEvent Pointer to the SVM event.
6570	*
6571	* @remarks This function currently only constructs flags required for
6572	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g. error-code
6573	* and CR2 aspects of an exception are not included).
6574	*/
6575	static uint32_t hmR0SvmGetIemXcptFlags(PCSVMEVENT pEvent)
6576	{
6577	uint8_t const uEventType = pEvent->n.u3Type;
6578	uint32_t fIemXcptFlags;
6579	switch (uEventType)
6580	{
6581	case SVM_EVENT_EXCEPTION:
6582	/*
6583	* Only INT3 and INTO instructions can raise #BP and #OF exceptions.
6584	* See AMD spec. Table 8-1. "Interrupt Vector Source and Cause".
6585	*/
6586	if (pEvent->n.u8Vector == X86_XCPT_BP)
6587	{
6588	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_BP_INSTR;
6589	break;
6590	}
6591	if (pEvent->n.u8Vector == X86_XCPT_OF)
6592	{
6593	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_OF_INSTR;
6594	break;
6595	}
6596	/** @todo How do we distinguish ICEBP \#DB from the regular one? */
6597	RT_FALL_THRU();
6598	case SVM_EVENT_NMI:
6599	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6600	break;
6601
6602	case SVM_EVENT_EXTERNAL_IRQ:
6603	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6604	break;
6605
6606	case SVM_EVENT_SOFTWARE_INT:
6607	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6608	break;
6609
6610	default:
6611	fIemXcptFlags = 0;
6612	AssertMsgFailed(("Unexpected event type! uEventType=%#x uVector=%#x", uEventType, pEvent->n.u8Vector));
6613	break;
6614	}
6615	return fIemXcptFlags;
6616	}
6617
6618
6619	/**
6620	* Handle a condition that occurred while delivering an event through the guest
6621	* IDT.
6622	*
6623	* @returns VBox status code (informational error codes included).
6624	* @retval VINF_SUCCESS if we should continue handling the \#VMEXIT.
6625	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought to
6626	* continue execution of the guest which will delivery the \#DF.
6627	* @retval VINF_EM_RESET if we detected a triple-fault condition.
6628	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
6629	*
6630	* @param pVCpu The cross context virtual CPU structure.
6631	* @param pSvmTransient Pointer to the SVM transient structure.
6632	*
6633	* @remarks No-long-jump zone!!!
6634	*/
6635	static int hmR0SvmCheckExitDueToEventDelivery(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6636	{
6637	/** @todo r=bird: Looks like this is called on many exits and we start by
6638	* loading CR2 on the offchance that we actually have work to do here.
6639	*
6640	* HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY can surely check
6641	* pVmcb->ctrl.ExitIntInfo.n.u1Valid, can't it?
6642	*
6643	* Also, what's the deal with hmR0SvmGetCurrentVmcb() vs pSvmTransient->pVmcb?
6644	*/
6645	int rc = VINF_SUCCESS;
6646	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6647	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR2);
6648
6649	Log4(("EXITINTINFO: Pending vectoring event %#RX64 Valid=%RTbool ErrValid=%RTbool Err=%#RX32 Type=%u Vector=%u\n",
6650	pVmcb->ctrl.ExitIntInfo.u, !!pVmcb->ctrl.ExitIntInfo.n.u1Valid, !!pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid,
6651	pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode, pVmcb->ctrl.ExitIntInfo.n.u3Type, pVmcb->ctrl.ExitIntInfo.n.u8Vector));
6652
6653	/*
6654	* The EXITINTINFO (if valid) contains the prior exception (IDT vector) that was trying to
6655	* be delivered to the guest which caused a #VMEXIT which was intercepted (Exit vector).
6656	*
6657	* See AMD spec. 15.7.3 "EXITINFO Pseudo-Code".
6658	*/
6659	if (pVmcb->ctrl.ExitIntInfo.n.u1Valid)
6660	{
6661	IEMXCPTRAISE enmRaise;
6662	IEMXCPTRAISEINFO fRaiseInfo;
6663	bool const fExitIsHwXcpt = pSvmTransient->u64ExitCode - SVM_EXIT_XCPT_0 <= SVM_EXIT_XCPT_31;
6664	uint8_t const uIdtVector = pVmcb->ctrl.ExitIntInfo.n.u8Vector;
6665	if (fExitIsHwXcpt)
6666	{
6667	uint8_t const uExitVector = pSvmTransient->u64ExitCode - SVM_EXIT_XCPT_0;
6668	uint32_t const fIdtVectorFlags = hmR0SvmGetIemXcptFlags(&pVmcb->ctrl.ExitIntInfo);
6669	uint32_t const fExitVectorFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6670	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
6671	}
6672	else
6673	{
6674	/*
6675	* If delivery of an event caused a #VMEXIT that is not an exception (e.g. #NPF)
6676	* then we end up here.
6677	*
6678	* If the event was:
6679	* - a software interrupt, we can re-execute the instruction which will
6680	* regenerate the event.
6681	* - an NMI, we need to clear NMI blocking and re-inject the NMI.
6682	* - a hardware exception or external interrupt, we re-inject it.
6683	*/
6684	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6685	if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_SOFTWARE_INT)
6686	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
6687	else
6688	enmRaise = IEMXCPTRAISE_PREV_EVENT;
6689	}
6690
6691	switch (enmRaise)
6692	{
6693	case IEMXCPTRAISE_CURRENT_XCPT:
6694	case IEMXCPTRAISE_PREV_EVENT:
6695	{
6696	/* For software interrupts, we shall re-execute the instruction. */
6697	if (!(fRaiseInfo & IEMXCPTRAISEINFO_SOFT_INT_XCPT))
6698	{
6699	RTGCUINTPTR GCPtrFaultAddress = 0;
6700
6701	/* If we are re-injecting an NMI, clear NMI blocking. */
6702	if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_NMI)
6703	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6704
6705	/* Determine a vectoring #PF condition, see comment in hmR0SvmExitXcptPF(). */
6706	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
6707	{
6708	pSvmTransient->fVectoringPF = true;
6709	Log4Func(("IDT: Pending vectoring #PF due to delivery of Ext-Int/NMI. uCR2=%#RX64\n",
6710	pVCpu->cpum.GstCtx.cr2));
6711	}
6712	else if ( pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_EXCEPTION
6713	&& uIdtVector == X86_XCPT_PF)
6714	{
6715	/*
6716	* If the previous exception was a #PF, we need to recover the CR2 value.
6717	* This can't happen with shadow paging.
6718	*/
6719	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
6720	}
6721
6722	/*
6723	* Without nested paging, when uExitVector is #PF, CR2 value will be updated from the VMCB's
6724	* exit info. fields, if it's a guest #PF, see hmR0SvmExitXcptPF().
6725	*/
6726	Assert(pVmcb->ctrl.ExitIntInfo.n.u3Type != SVM_EVENT_SOFTWARE_INT);
6727	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflect);
6728	hmR0SvmSetPendingEvent(pVCpu, &pVmcb->ctrl.ExitIntInfo, GCPtrFaultAddress);
6729
6730	Log4Func(("IDT: Pending vectoring event %#RX64 ErrValid=%RTbool Err=%#RX32 GCPtrFaultAddress=%#RX64\n",
6731	pVmcb->ctrl.ExitIntInfo.u, RT_BOOL(pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid),
6732	pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode, GCPtrFaultAddress));
6733	}
6734	break;
6735	}
6736
6737	case IEMXCPTRAISE_REEXEC_INSTR:
6738	{
6739	Assert(rc == VINF_SUCCESS);
6740	break;
6741	}
6742
6743	case IEMXCPTRAISE_DOUBLE_FAULT:
6744	{
6745	/*
6746	* Determing a vectoring double #PF condition. Used later, when PGM evaluates
6747	* the second #PF as a guest #PF (and not a shadow #PF) and needs to be
6748	* converted into a #DF.
6749	*/
6750	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6751	{
6752	Log4Func(("IDT: Pending vectoring double #PF uCR2=%#RX64\n", pVCpu->cpum.GstCtx.cr2));
6753	pSvmTransient->fVectoringDoublePF = true;
6754	Assert(rc == VINF_SUCCESS);
6755	}
6756	else
6757	{
6758	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectConvertDF);
6759	hmR0SvmSetPendingXcptDF(pVCpu);
6760	rc = VINF_HM_DOUBLE_FAULT;
6761	}
6762	break;
6763	}
6764
6765	case IEMXCPTRAISE_TRIPLE_FAULT:
6766	{
6767	rc = VINF_EM_RESET;
6768	break;
6769	}
6770
6771	case IEMXCPTRAISE_CPU_HANG:
6772	{
6773	rc = VERR_EM_GUEST_CPU_HANG;
6774	break;
6775	}
6776
6777	default:
6778	AssertMsgFailedBreakStmt(("Bogus enmRaise value: %d (%#x)\n", enmRaise, enmRaise), rc = VERR_SVM_IPE_2);
6779	}
6780	}
6781	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET \|\| rc == VERR_EM_GUEST_CPU_HANG);
6782	return rc;
6783	}
6784
6785
6786	/**
6787	* Advances the guest RIP by the number of bytes specified in @a cb.
6788	*
6789	* @param pVCpu The cross context virtual CPU structure.
6790	* @param cb RIP increment value in bytes.
6791	*/
6792	DECLINLINE(void) hmR0SvmAdvanceRip(PVMCPUCC pVCpu, uint32_t cb)
6793	{
6794	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6795	pCtx->rip += cb;
6796
6797	/* Update interrupt shadow. */
6798	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
6799	&& pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
6800	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6801	}
6802
6803
6804	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6805	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- #VMEXIT handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6806	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6807
6808	/** @name \#VMEXIT handlers.
6809	* @{
6810	*/
6811
6812	/**
6813	* \#VMEXIT handler for external interrupts, NMIs, FPU assertion freeze and INIT
6814	* signals (SVM_EXIT_INTR, SVM_EXIT_NMI, SVM_EXIT_FERR_FREEZE, SVM_EXIT_INIT).
6815	*/
6816	HMSVM_EXIT_DECL hmR0SvmExitIntr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6817	{
6818	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6819
6820	if (pSvmTransient->u64ExitCode == SVM_EXIT_NMI)
6821	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
6822	else if (pSvmTransient->u64ExitCode == SVM_EXIT_INTR)
6823	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6824
6825	/*
6826	* AMD-V has no preemption timer and the generic periodic preemption timer has no way to
6827	* signal -before- the timer fires if the current interrupt is our own timer or a some
6828	* other host interrupt. We also cannot examine what interrupt it is until the host
6829	* actually take the interrupt.
6830	*
6831	* Going back to executing guest code here unconditionally causes random scheduling
6832	* problems (observed on an AMD Phenom 9850 Quad-Core on Windows 64-bit host).
6833	*/
6834	return VINF_EM_RAW_INTERRUPT;
6835	}
6836
6837
6838	/**
6839	* \#VMEXIT handler for WBINVD (SVM_EXIT_WBINVD). Conditional \#VMEXIT.
6840	*/
6841	HMSVM_EXIT_DECL hmR0SvmExitWbinvd(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6842	{
6843	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6844
6845	VBOXSTRICTRC rcStrict;
6846	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6847	if (fSupportsNextRipSave)
6848	{
6849	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
6850	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6851	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6852	rcStrict = IEMExecDecodedWbinvd(pVCpu, cbInstr);
6853	}
6854	else
6855	{
6856	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6857	rcStrict = IEMExecOne(pVCpu);
6858	}
6859
6860	if (rcStrict == VINF_IEM_RAISED_XCPT)
6861	{
6862	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6863	rcStrict = VINF_SUCCESS;
6864	}
6865	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6866	return rcStrict;
6867	}
6868
6869
6870	/**
6871	* \#VMEXIT handler for INVD (SVM_EXIT_INVD). Unconditional \#VMEXIT.
6872	*/
6873	HMSVM_EXIT_DECL hmR0SvmExitInvd(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6874	{
6875	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6876
6877	VBOXSTRICTRC rcStrict;
6878	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6879	if (fSupportsNextRipSave)
6880	{
6881	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
6882	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6883	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6884	rcStrict = IEMExecDecodedInvd(pVCpu, cbInstr);
6885	}
6886	else
6887	{
6888	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6889	rcStrict = IEMExecOne(pVCpu);
6890	}
6891
6892	if (rcStrict == VINF_IEM_RAISED_XCPT)
6893	{
6894	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6895	rcStrict = VINF_SUCCESS;
6896	}
6897	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6898	return rcStrict;
6899	}
6900
6901
6902	/**
6903	* \#VMEXIT handler for INVD (SVM_EXIT_CPUID). Conditional \#VMEXIT.
6904	*/
6905	HMSVM_EXIT_DECL hmR0SvmExitCpuid(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6906	{
6907	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6908
6909	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX);
6910	VBOXSTRICTRC rcStrict;
6911	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
6912	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
6913	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
6914	if (!pExitRec)
6915	{
6916	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6917	if (fSupportsNextRipSave)
6918	{
6919	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6920	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6921	rcStrict = IEMExecDecodedCpuid(pVCpu, cbInstr);
6922	}
6923	else
6924	{
6925	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6926	rcStrict = IEMExecOne(pVCpu);
6927	}
6928
6929	if (rcStrict == VINF_IEM_RAISED_XCPT)
6930	{
6931	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6932	rcStrict = VINF_SUCCESS;
6933	}
6934	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6935	}
6936	else
6937	{
6938	/*
6939	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
6940	*/
6941	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6942
6943	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
6944	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
6945
6946	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
6947
6948	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
6949	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
6950	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
6951	}
6952	return rcStrict;
6953	}
6954
6955
6956	/**
6957	* \#VMEXIT handler for RDTSC (SVM_EXIT_RDTSC). Conditional \#VMEXIT.
6958	*/
6959	HMSVM_EXIT_DECL hmR0SvmExitRdtsc(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6960	{
6961	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6962
6963	VBOXSTRICTRC rcStrict;
6964	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6965	if (fSupportsNextRipSave)
6966	{
6967	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4);
6968	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6969	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6970	rcStrict = IEMExecDecodedRdtsc(pVCpu, cbInstr);
6971	}
6972	else
6973	{
6974	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6975	rcStrict = IEMExecOne(pVCpu);
6976	}
6977
6978	if (rcStrict == VINF_SUCCESS)
6979	pSvmTransient->fUpdateTscOffsetting = true;
6980	else if (rcStrict == VINF_IEM_RAISED_XCPT)
6981	{
6982	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6983	rcStrict = VINF_SUCCESS;
6984	}
6985	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6986	return rcStrict;
6987	}
6988
6989
6990	/**
6991	* \#VMEXIT handler for RDTSCP (SVM_EXIT_RDTSCP). Conditional \#VMEXIT.
6992	*/
6993	HMSVM_EXIT_DECL hmR0SvmExitRdtscp(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6994	{
6995	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6996
6997	VBOXSTRICTRC rcStrict;
6998	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6999	if (fSupportsNextRipSave)
7000	{
7001	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_TSC_AUX);
7002	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7003	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7004	rcStrict = IEMExecDecodedRdtscp(pVCpu, cbInstr);
7005	}
7006	else
7007	{
7008	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7009	rcStrict = IEMExecOne(pVCpu);
7010	}
7011
7012	if (rcStrict == VINF_SUCCESS)
7013	pSvmTransient->fUpdateTscOffsetting = true;
7014	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7015	{
7016	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7017	rcStrict = VINF_SUCCESS;
7018	}
7019	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7020	return rcStrict;
7021	}
7022
7023
7024	/**
7025	* \#VMEXIT handler for RDPMC (SVM_EXIT_RDPMC). Conditional \#VMEXIT.
7026	*/
7027	HMSVM_EXIT_DECL hmR0SvmExitRdpmc(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7028	{
7029	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7030
7031	VBOXSTRICTRC rcStrict;
7032	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7033	if (fSupportsNextRipSave)
7034	{
7035	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4);
7036	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7037	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7038	rcStrict = IEMExecDecodedRdpmc(pVCpu, cbInstr);
7039	}
7040	else
7041	{
7042	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7043	rcStrict = IEMExecOne(pVCpu);
7044	}
7045
7046	if (rcStrict == VINF_IEM_RAISED_XCPT)
7047	{
7048	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7049	rcStrict = VINF_SUCCESS;
7050	}
7051	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7052	return rcStrict;
7053	}
7054
7055
7056	/**
7057	* \#VMEXIT handler for INVLPG (SVM_EXIT_INVLPG). Conditional \#VMEXIT.
7058	*/
7059	HMSVM_EXIT_DECL hmR0SvmExitInvlpg(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7060	{
7061	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7062	Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
7063
7064	VBOXSTRICTRC rcStrict;
7065	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
7066	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7067	if ( fSupportsDecodeAssists
7068	&& fSupportsNextRipSave)
7069	{
7070	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
7071	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7072	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7073	RTGCPTR const GCPtrPage = pVmcb->ctrl.u64ExitInfo1;
7074	rcStrict = IEMExecDecodedInvlpg(pVCpu, cbInstr, GCPtrPage);
7075	}
7076	else
7077	{
7078	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7079	rcStrict = IEMExecOne(pVCpu);
7080	}
7081
7082	if (rcStrict == VINF_IEM_RAISED_XCPT)
7083	{
7084	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7085	rcStrict = VINF_SUCCESS;
7086	}
7087	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7088	return VBOXSTRICTRC_VAL(rcStrict);
7089	}
7090
7091
7092	/**
7093	* \#VMEXIT handler for HLT (SVM_EXIT_HLT). Conditional \#VMEXIT.
7094	*/
7095	HMSVM_EXIT_DECL hmR0SvmExitHlt(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7096	{
7097	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7098
7099	VBOXSTRICTRC rcStrict;
7100	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7101	if (fSupportsNextRipSave)
7102	{
7103	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
7104	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7105	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7106	rcStrict = IEMExecDecodedHlt(pVCpu, cbInstr);
7107	}
7108	else
7109	{
7110	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7111	rcStrict = IEMExecOne(pVCpu);
7112	}
7113
7114	if ( rcStrict == VINF_EM_HALT
7115	\|\| rcStrict == VINF_SUCCESS)
7116	rcStrict = EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx) ? VINF_SUCCESS : VINF_EM_HALT;
7117	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7118	{
7119	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7120	rcStrict = VINF_SUCCESS;
7121	}
7122	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7123	if (rcStrict != VINF_SUCCESS)
7124	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
7125	return VBOXSTRICTRC_VAL(rcStrict);;
7126	}
7127
7128
7129	/**
7130	* \#VMEXIT handler for MONITOR (SVM_EXIT_MONITOR). Conditional \#VMEXIT.
7131	*/
7132	HMSVM_EXIT_DECL hmR0SvmExitMonitor(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7133	{
7134	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7135
7136	/*
7137	* If the instruction length is supplied by the CPU is 3 bytes, we can be certain that no
7138	* segment override prefix is present (and thus use the default segment DS). Otherwise, a
7139	* segment override prefix or other prefixes might be used, in which case we fallback to
7140	* IEMExecOne() to figure out.
7141	*/
7142	VBOXSTRICTRC rcStrict;
7143	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7144	uint8_t const cbInstr = hmR0SvmSupportsNextRipSave(pVCpu) ? pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip : 0;
7145	if (cbInstr)
7146	{
7147	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
7148	rcStrict = IEMExecDecodedMonitor(pVCpu, cbInstr);
7149	}
7150	else
7151	{
7152	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7153	rcStrict = IEMExecOne(pVCpu);
7154	}
7155
7156	if (rcStrict == VINF_IEM_RAISED_XCPT)
7157	{
7158	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7159	rcStrict = VINF_SUCCESS;
7160	}
7161	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7162	return rcStrict;
7163	}
7164
7165
7166	/**
7167	* \#VMEXIT handler for MWAIT (SVM_EXIT_MWAIT). Conditional \#VMEXIT.
7168	*/
7169	HMSVM_EXIT_DECL hmR0SvmExitMwait(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7170	{
7171	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7172
7173	VBOXSTRICTRC rcStrict;
7174	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7175	if (fSupportsNextRipSave)
7176	{
7177	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
7178	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7179	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7180	rcStrict = IEMExecDecodedMwait(pVCpu, cbInstr);
7181	}
7182	else
7183	{
7184	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7185	rcStrict = IEMExecOne(pVCpu);
7186	}
7187
7188	if ( rcStrict == VINF_EM_HALT
7189	&& EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
7190	rcStrict = VINF_SUCCESS;
7191	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7192	{
7193	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7194	rcStrict = VINF_SUCCESS;
7195	}
7196	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7197	return rcStrict;
7198	}
7199
7200
7201	/**
7202	* \#VMEXIT handler for shutdown (triple-fault) (SVM_EXIT_SHUTDOWN). Conditional
7203	* \#VMEXIT.
7204	*/
7205	HMSVM_EXIT_DECL hmR0SvmExitShutdown(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7206	{
7207	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7208	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7209	return VINF_EM_RESET;
7210	}
7211
7212
7213	/**
7214	* \#VMEXIT handler for unexpected exits. Conditional \#VMEXIT.
7215	*/
7216	HMSVM_EXIT_DECL hmR0SvmExitUnexpected(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7217	{
7218	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7219	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7220	AssertMsgFailed(("hmR0SvmExitUnexpected: ExitCode=%#RX64 uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", pSvmTransient->u64ExitCode,
7221	pVmcb->ctrl.u64ExitInfo1, pVmcb->ctrl.u64ExitInfo2));
7222	RT_NOREF(pVmcb);
7223	pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
7224	return VERR_SVM_UNEXPECTED_EXIT;
7225	}
7226
7227
7228	/**
7229	* \#VMEXIT handler for CRx reads (SVM_EXIT_READ_CR*). Conditional \#VMEXIT.
7230	*/
7231	HMSVM_EXIT_DECL hmR0SvmExitReadCRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7232	{
7233	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7234
7235	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7236	Log4Func(("CS:RIP=%04x:%RX64\n", pCtx->cs.Sel, pCtx->rip));
7237	#ifdef VBOX_WITH_STATISTICS
7238	switch (pSvmTransient->u64ExitCode)
7239	{
7240	case SVM_EXIT_READ_CR0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
7241	case SVM_EXIT_READ_CR2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
7242	case SVM_EXIT_READ_CR3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
7243	case SVM_EXIT_READ_CR4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
7244	case SVM_EXIT_READ_CR8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
7245	}
7246	#endif
7247
7248	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
7249	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7250	if ( fSupportsDecodeAssists
7251	&& fSupportsNextRipSave)
7252	{
7253	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7254	bool const fMovCRx = RT_BOOL(pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_MASK);
7255	if (fMovCRx)
7256	{
7257	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR_MASK
7258	\| CPUMCTX_EXTRN_APIC_TPR);
7259	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
7260	uint8_t const iCrReg = pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0;
7261	uint8_t const iGReg = pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_GPR_NUMBER;
7262	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
7263	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7264	return VBOXSTRICTRC_VAL(rcStrict);
7265	}
7266	/* else: SMSW instruction, fall back below to IEM for this. */
7267	}
7268
7269	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7270	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
7271	AssertMsg( rcStrict == VINF_SUCCESS
7272	\|\| rcStrict == VINF_PGM_SYNC_CR3
7273	\|\| rcStrict == VINF_IEM_RAISED_XCPT,
7274	("hmR0SvmExitReadCRx: IEMExecOne failed rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7275	Assert((pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0) <= 15);
7276	if (rcStrict == VINF_IEM_RAISED_XCPT)
7277	{
7278	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7279	rcStrict = VINF_SUCCESS;
7280	}
7281	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7282	return rcStrict;
7283	}
7284
7285
7286	/**
7287	* \#VMEXIT handler for CRx writes (SVM_EXIT_WRITE_CR*). Conditional \#VMEXIT.
7288	*/
7289	HMSVM_EXIT_DECL hmR0SvmExitWriteCRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7290	{
7291	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7292
7293	uint64_t const uExitCode = pSvmTransient->u64ExitCode;
7294	uint8_t const iCrReg = uExitCode == SVM_EXIT_CR0_SEL_WRITE ? 0 : (pSvmTransient->u64ExitCode - SVM_EXIT_WRITE_CR0);
7295	Assert(iCrReg <= 15);
7296
7297	VBOXSTRICTRC rcStrict = VERR_SVM_IPE_5;
7298	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7299	bool fDecodedInstr = false;
7300	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
7301	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7302	if ( fSupportsDecodeAssists
7303	&& fSupportsNextRipSave)
7304	{
7305	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7306	bool const fMovCRx = RT_BOOL(pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_MASK);
7307	if (fMovCRx)
7308	{
7309	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4
7310	\| CPUMCTX_EXTRN_APIC_TPR);
7311	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
7312	uint8_t const iGReg = pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_GPR_NUMBER;
7313	Log4Func(("Mov CR%u w/ iGReg=%#x\n", iCrReg, iGReg));
7314	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
7315	fDecodedInstr = true;
7316	}
7317	/* else: LMSW or CLTS instruction, fall back below to IEM for this. */
7318	}
7319
7320	if (!fDecodedInstr)
7321	{
7322	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7323	Log4Func(("iCrReg=%#x\n", iCrReg));
7324	rcStrict = IEMExecOne(pVCpu);
7325	if (RT_UNLIKELY( rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED
7326	\|\| rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED))
7327	rcStrict = VERR_EM_INTERPRETER;
7328	}
7329
7330	if (rcStrict == VINF_SUCCESS)
7331	{
7332	switch (iCrReg)
7333	{
7334	case 0:
7335	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
7336	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
7337	break;
7338
7339	case 2:
7340	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR2);
7341	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
7342	break;
7343
7344	case 3:
7345	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR3);
7346	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
7347	break;
7348
7349	case 4:
7350	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
7351	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
7352	break;
7353
7354	case 8:
7355	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7356	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
7357	break;
7358
7359	default:
7360	{
7361	AssertMsgFailed(("hmR0SvmExitWriteCRx: Invalid/Unexpected Write-CRx exit. u64ExitCode=%#RX64 %#x\n",
7362	pSvmTransient->u64ExitCode, iCrReg));
7363	break;
7364	}
7365	}
7366	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7367	}
7368	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7369	{
7370	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7371	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7372	rcStrict = VINF_SUCCESS;
7373	}
7374	else
7375	Assert(rcStrict == VERR_EM_INTERPRETER \|\| rcStrict == VINF_PGM_SYNC_CR3);
7376	return rcStrict;
7377	}
7378
7379
7380	/**
7381	* \#VMEXIT helper for read MSRs, see hmR0SvmExitMsr.
7382	*
7383	* @returns Strict VBox status code.
7384	* @param pVCpu The cross context virtual CPU structure.
7385	* @param pVmcb Pointer to the VM control block.
7386	*/
7387	static VBOXSTRICTRC hmR0SvmExitReadMsr(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
7388	{
7389	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7390	Log4Func(("idMsr=%#RX32\n", pVCpu->cpum.GstCtx.ecx));
7391
7392	VBOXSTRICTRC rcStrict;
7393	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7394	if (fSupportsNextRipSave)
7395	{
7396	/** @todo Optimize this: Only retrieve the MSR bits we need here. CPUMAllMsrs.cpp
7397	* can ask for what it needs instead of using CPUMCTX_EXTRN_ALL_MSRS. */
7398	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
7399	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7400	rcStrict = IEMExecDecodedRdmsr(pVCpu, cbInstr);
7401	}
7402	else
7403	{
7404	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
7405	rcStrict = IEMExecOne(pVCpu);
7406	}
7407
7408	AssertMsg( rcStrict == VINF_SUCCESS
7409	\|\| rcStrict == VINF_IEM_RAISED_XCPT
7410	\|\| rcStrict == VINF_CPUM_R3_MSR_READ,
7411	("hmR0SvmExitReadMsr: Unexpected status %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7412
7413	if (rcStrict == VINF_IEM_RAISED_XCPT)
7414	{
7415	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7416	rcStrict = VINF_SUCCESS;
7417	}
7418	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7419	return rcStrict;
7420	}
7421
7422
7423	/**
7424	* \#VMEXIT helper for write MSRs, see hmR0SvmExitMsr.
7425	*
7426	* @returns Strict VBox status code.
7427	* @param pVCpu The cross context virtual CPU structure.
7428	* @param pVmcb Pointer to the VM control block.
7429	* @param pSvmTransient Pointer to the SVM-transient structure.
7430	*/
7431	static VBOXSTRICTRC hmR0SvmExitWriteMsr(PVMCPUCC pVCpu, PSVMVMCB pVmcb, PSVMTRANSIENT pSvmTransient)
7432	{
7433	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7434	uint32_t const idMsr = pCtx->ecx;
7435	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7436	Log4Func(("idMsr=%#RX32\n", idMsr));
7437
7438	/*
7439	* Handle TPR patching MSR writes.
7440	* We utilitize the LSTAR MSR for patching.
7441	*/
7442	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7443	if ( idMsr == MSR_K8_LSTAR
7444	&& pVCpu->CTX_SUFF(pVM)->hm.s.fTprPatchingActive)
7445	{
7446	unsigned cbInstr;
7447	if (fSupportsNextRipSave)
7448	cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7449	else
7450	{
7451	PDISCPUSTATE pDis = &pVCpu->hmr0.s.svm.DisState;
7452	int rc = EMInterpretDisasCurrent(pVCpu->CTX_SUFF(pVM), pVCpu, pDis, &cbInstr);
7453	if ( rc == VINF_SUCCESS
7454	&& pDis->pCurInstr->uOpcode == OP_WRMSR)
7455	Assert(cbInstr > 0);
7456	else
7457	cbInstr = 0;
7458	}
7459
7460	/* Our patch code uses LSTAR for TPR caching for 32-bit guests. */
7461	if ((pCtx->eax & 0xff) != pSvmTransient->u8GuestTpr)
7462	{
7463	int rc = APICSetTpr(pVCpu, pCtx->eax & 0xff);
7464	AssertRCReturn(rc, rc);
7465	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7466	}
7467
7468	int rc = VINF_SUCCESS;
7469	hmR0SvmAdvanceRip(pVCpu, cbInstr);
7470	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
7471	return rc;
7472	}
7473
7474	/*
7475	* Handle regular MSR writes.
7476	*/
7477	VBOXSTRICTRC rcStrict;
7478	if (fSupportsNextRipSave)
7479	{
7480	/** @todo Optimize this: We don't need to get much of the MSR state here
7481	* since we're only updating. CPUMAllMsrs.cpp can ask for what it needs and
7482	* clear the applicable extern flags. */
7483	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
7484	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7485	rcStrict = IEMExecDecodedWrmsr(pVCpu, cbInstr);
7486	}
7487	else
7488	{
7489	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
7490	rcStrict = IEMExecOne(pVCpu);
7491	}
7492
7493	AssertMsg( rcStrict == VINF_SUCCESS
7494	\|\| rcStrict == VINF_IEM_RAISED_XCPT
7495	\|\| rcStrict == VINF_CPUM_R3_MSR_WRITE,
7496	("hmR0SvmExitWriteMsr: Unexpected status %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7497
7498	if (rcStrict == VINF_SUCCESS)
7499	{
7500	/* If this is an X2APIC WRMSR access, update the APIC TPR state. */
7501	if ( idMsr >= MSR_IA32_X2APIC_START
7502	&& idMsr <= MSR_IA32_X2APIC_END)
7503	{
7504	/*
7505	* We've already saved the APIC related guest-state (TPR) in hmR0SvmPostRunGuest().
7506	* When full APIC register virtualization is implemented we'll have to make sure
7507	* APIC state is saved from the VMCB before IEM changes it.
7508	*/
7509	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7510	}
7511	else
7512	{
7513	switch (idMsr)
7514	{
7515	case MSR_IA32_TSC: pSvmTransient->fUpdateTscOffsetting = true; break;
7516	case MSR_K6_EFER: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR); break;
7517	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
7518	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
7519	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
7520	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
7521	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
7522	}
7523	}
7524	}
7525	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7526	{
7527	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7528	rcStrict = VINF_SUCCESS;
7529	}
7530	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7531	return rcStrict;
7532	}
7533
7534
7535	/**
7536	* \#VMEXIT handler for MSR read and writes (SVM_EXIT_MSR). Conditional
7537	* \#VMEXIT.
7538	*/
7539	HMSVM_EXIT_DECL hmR0SvmExitMsr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7540	{
7541	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7542
7543	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7544	if (pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_READ)
7545	return hmR0SvmExitReadMsr(pVCpu, pVmcb);
7546
7547	Assert(pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_WRITE);
7548	return hmR0SvmExitWriteMsr(pVCpu, pVmcb, pSvmTransient);
7549	}
7550
7551
7552	/**
7553	* \#VMEXIT handler for DRx read (SVM_EXIT_READ_DRx). Conditional \#VMEXIT.
7554	*/
7555	HMSVM_EXIT_DECL hmR0SvmExitReadDRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7556	{
7557	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7558	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7559
7560	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
7561
7562	/** @todo Stepping with nested-guest. */
7563	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7564	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
7565	{
7566	/* We should -not- get this #VMEXIT if the guest's debug registers were active. */
7567	if (pSvmTransient->fWasGuestDebugStateActive)
7568	{
7569	AssertMsgFailed(("hmR0SvmExitReadDRx: Unexpected exit %#RX32\n", (uint32_t)pSvmTransient->u64ExitCode));
7570	pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
7571	return VERR_SVM_UNEXPECTED_EXIT;
7572	}
7573
7574	/*
7575	* Lazy DR0-3 loading.
7576	*/
7577	if (!pSvmTransient->fWasHyperDebugStateActive)
7578	{
7579	Assert(!DBGFIsStepping(pVCpu)); Assert(!pVCpu->hm.s.fSingleInstruction);
7580	Log5(("hmR0SvmExitReadDRx: Lazy loading guest debug registers\n"));
7581
7582	/* Don't intercept DRx read and writes. */
7583	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
7584	pVmcb->ctrl.u16InterceptRdDRx = 0;
7585	pVmcb->ctrl.u16InterceptWrDRx = 0;
7586	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
7587
7588	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
7589	VMMRZCallRing3Disable(pVCpu);
7590	HM_DISABLE_PREEMPT(pVCpu);
7591
7592	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
7593	CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
7594	Assert(CPUMIsGuestDebugStateActive(pVCpu));
7595
7596	HM_RESTORE_PREEMPT();
7597	VMMRZCallRing3Enable(pVCpu);
7598
7599	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
7600	return VINF_SUCCESS;
7601	}
7602	}
7603
7604	/*
7605	* Interpret the read/writing of DRx.
7606	*/
7607	/** @todo Decode assist. */
7608	VBOXSTRICTRC rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0 /* pvFault */);
7609	Log5(("hmR0SvmExitReadDRx: Emulated DRx access: rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
7610	if (RT_LIKELY(rc == VINF_SUCCESS))
7611	{
7612	/* Not necessary for read accesses but whatever doesn't hurt for now, will be fixed with decode assist. */
7613	/** @todo CPUM should set this flag! */
7614	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR_MASK);
7615	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
7616	}
7617	else
7618	Assert(rc == VERR_EM_INTERPRETER);
7619	return rc;
7620	}
7621
7622
7623	/**
7624	* \#VMEXIT handler for DRx write (SVM_EXIT_WRITE_DRx). Conditional \#VMEXIT.
7625	*/
7626	HMSVM_EXIT_DECL hmR0SvmExitWriteDRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7627	{
7628	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7629	/* For now it's the same since we interpret the instruction anyway. Will change when using of Decode Assist is implemented. */
7630	VBOXSTRICTRC rc = hmR0SvmExitReadDRx(pVCpu, pSvmTransient);
7631	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
7632	STAM_COUNTER_DEC(&pVCpu->hm.s.StatExitDRxRead);
7633	return rc;
7634	}
7635
7636
7637	/**
7638	* \#VMEXIT handler for XCRx write (SVM_EXIT_XSETBV). Conditional \#VMEXIT.
7639	*/
7640	HMSVM_EXIT_DECL hmR0SvmExitXsetbv(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7641	{
7642	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7643	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7644
7645	/** @todo decode assists... */
7646	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
7647	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
7648	{
7649	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7650	bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
7651	Log4Func(("New XCR0=%#RX64 fLoadSaveGuestXcr0=%RTbool (cr4=%#RX64)\n", pCtx->aXcr[0], fLoadSaveGuestXcr0, pCtx->cr4));
7652	if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
7653	{
7654	pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
7655	hmR0SvmUpdateVmRunFunction(pVCpu);
7656	}
7657	}
7658	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7659	{
7660	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7661	rcStrict = VINF_SUCCESS;
7662	}
7663	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7664	return rcStrict;
7665	}
7666
7667
7668	/**
7669	* \#VMEXIT handler for I/O instructions (SVM_EXIT_IOIO). Conditional \#VMEXIT.
7670	*/
7671	HMSVM_EXIT_DECL hmR0SvmExitIOInstr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7672	{
7673	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7674	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK);
7675
7676	/* I/O operation lookup arrays. */
7677	static uint32_t const s_aIOSize[8] = { 0, 1, 2, 0, 4, 0, 0, 0 }; /* Size of the I/O accesses in bytes. */
7678	static uint32_t const s_aIOOpAnd[8] = { 0, 0xff, 0xffff, 0, 0xffffffff, 0, 0, 0 }; /* AND masks for saving
7679	the result (in AL/AX/EAX). */
7680	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7681	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7682	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7683
7684	Log4Func(("CS:RIP=%04x:%RX64\n", pCtx->cs.Sel, pCtx->rip));
7685
7686	/* Refer AMD spec. 15.10.2 "IN and OUT Behaviour" and Figure 15-2. "EXITINFO1 for IOIO Intercept" for the format. */
7687	SVMIOIOEXITINFO IoExitInfo;
7688	IoExitInfo.u = (uint32_t)pVmcb->ctrl.u64ExitInfo1;
7689	uint32_t uIOWidth = (IoExitInfo.u >> 4) & 0x7;
7690	uint32_t cbValue = s_aIOSize[uIOWidth];
7691	uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
7692
7693	if (RT_UNLIKELY(!cbValue))
7694	{
7695	AssertMsgFailed(("hmR0SvmExitIOInstr: Invalid IO operation. uIOWidth=%u\n", uIOWidth));
7696	return VERR_EM_INTERPRETER;
7697	}
7698
7699	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
7700	VBOXSTRICTRC rcStrict;
7701	PCEMEXITREC pExitRec = NULL;
7702	if ( !pVCpu->hm.s.fSingleInstruction
7703	&& !pVCpu->cpum.GstCtx.eflags.Bits.u1TF)
7704	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
7705	!IoExitInfo.n.u1Str
7706	? IoExitInfo.n.u1Type == SVM_IOIO_READ
7707	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
7708	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
7709	: IoExitInfo.n.u1Type == SVM_IOIO_READ
7710	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
7711	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
7712	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
7713	if (!pExitRec)
7714	{
7715	bool fUpdateRipAlready = false;
7716	if (IoExitInfo.n.u1Str)
7717	{
7718	/* INS/OUTS - I/O String instruction. */
7719	/** @todo Huh? why can't we use the segment prefix information given by AMD-V
7720	* in EXITINFO1? Investigate once this thing is up and running. */
7721	Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, IoExitInfo.n.u16Port, cbValue,
7722	IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? 'w' : 'r'));
7723	AssertReturn(pCtx->dx == IoExitInfo.n.u16Port, VERR_SVM_IPE_2);
7724	static IEMMODE const s_aenmAddrMode[8] =
7725	{
7726	(IEMMODE)-1, IEMMODE_16BIT, IEMMODE_32BIT, (IEMMODE)-1, IEMMODE_64BIT, (IEMMODE)-1, (IEMMODE)-1, (IEMMODE)-1
7727	};
7728	IEMMODE enmAddrMode = s_aenmAddrMode[(IoExitInfo.u >> 7) & 0x7];
7729	if (enmAddrMode != (IEMMODE)-1)
7730	{
7731	uint64_t cbInstr = pVmcb->ctrl.u64ExitInfo2 - pCtx->rip;
7732	if (cbInstr <= 15 && cbInstr >= 1)
7733	{
7734	Assert(cbInstr >= 1U + IoExitInfo.n.u1Rep);
7735	if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
7736	{
7737	/* Don't know exactly how to detect whether u3Seg is valid, currently
7738	only enabling it for Bulldozer and later with NRIP. OS/2 broke on
7739	2384 Opterons when only checking NRIP. */
7740	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7741	if ( fSupportsNextRipSave
7742	&& pVM->cpum.ro.GuestFeatures.enmMicroarch >= kCpumMicroarch_AMD_15h_First)
7743	{
7744	AssertMsg(IoExitInfo.n.u3Seg == X86_SREG_DS \|\| cbInstr > 1U + IoExitInfo.n.u1Rep,
7745	("u32Seg=%d cbInstr=%d u1REP=%d", IoExitInfo.n.u3Seg, cbInstr, IoExitInfo.n.u1Rep));
7746	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
7747	IoExitInfo.n.u3Seg, true /fIoChecked/);
7748	}
7749	else if (cbInstr == 1U + IoExitInfo.n.u1Rep)
7750	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
7751	X86_SREG_DS, true /fIoChecked/);
7752	else
7753	rcStrict = IEMExecOne(pVCpu);
7754	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7755	}
7756	else
7757	{
7758	AssertMsg(IoExitInfo.n.u3Seg == X86_SREG_ES /=0/, ("%#x\n", IoExitInfo.n.u3Seg));
7759	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
7760	true /fIoChecked/);
7761	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7762	}
7763	}
7764	else
7765	{
7766	AssertMsgFailed(("rip=%RX64 nrip=%#RX64 cbInstr=%#RX64\n", pCtx->rip, pVmcb->ctrl.u64ExitInfo2, cbInstr));
7767	rcStrict = IEMExecOne(pVCpu);
7768	}
7769	}
7770	else
7771	{
7772	AssertMsgFailed(("IoExitInfo=%RX64\n", IoExitInfo.u));
7773	rcStrict = IEMExecOne(pVCpu);
7774	}
7775	fUpdateRipAlready = true;
7776	if (rcStrict == VINF_IEM_RAISED_XCPT)
7777	{
7778	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7779	rcStrict = VINF_SUCCESS;
7780	}
7781	}
7782	else
7783	{
7784	/* IN/OUT - I/O instruction. */
7785	Assert(!IoExitInfo.n.u1Rep);
7786
7787	uint8_t const cbInstr = pVmcb->ctrl.u64ExitInfo2 - pCtx->rip;
7788	if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
7789	{
7790	rcStrict = IOMIOPortWrite(pVM, pVCpu, IoExitInfo.n.u16Port, pCtx->eax & uAndVal, cbValue);
7791	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
7792	&& !pCtx->eflags.Bits.u1TF)
7793	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, IoExitInfo.n.u16Port, cbInstr, cbValue, pCtx->eax & uAndVal);
7794	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
7795	}
7796	else
7797	{
7798	uint32_t u32Val = 0;
7799	rcStrict = IOMIOPortRead(pVM, pVCpu, IoExitInfo.n.u16Port, &u32Val, cbValue);
7800	if (IOM_SUCCESS(rcStrict))
7801	{
7802	/* Save result of I/O IN instr. in AL/AX/EAX. */
7803	/** @todo r=bird: 32-bit op size should clear high bits of rax! */
7804	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
7805	}
7806	else if ( rcStrict == VINF_IOM_R3_IOPORT_READ
7807	&& !pCtx->eflags.Bits.u1TF)
7808	rcStrict = EMRZSetPendingIoPortRead(pVCpu, IoExitInfo.n.u16Port, cbInstr, cbValue);
7809
7810	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
7811	}
7812	}
7813
7814	if (IOM_SUCCESS(rcStrict))
7815	{
7816	/* AMD-V saves the RIP of the instruction following the IO instruction in EXITINFO2. */
7817	if (!fUpdateRipAlready)
7818	pCtx->rip = pVmcb->ctrl.u64ExitInfo2;
7819
7820	/*
7821	* If any I/O breakpoints are armed, we need to check if one triggered
7822	* and take appropriate action.
7823	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
7824	*/
7825	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
7826	* execution engines about whether hyper BPs and such are pending. */
7827	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_DR7);
7828	uint32_t const uDr7 = pCtx->dr[7];
7829	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
7830	&& X86_DR7_ANY_RW_IO(uDr7)
7831	&& (pCtx->cr4 & X86_CR4_DE))
7832	\|\| DBGFBpIsHwIoArmed(pVM)))
7833	{
7834	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
7835	VMMRZCallRing3Disable(pVCpu);
7836	HM_DISABLE_PREEMPT(pVCpu);
7837
7838	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
7839	CPUMR0DebugStateMaybeSaveGuest(pVCpu, false /fDr6/);
7840
7841	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, &pVCpu->cpum.GstCtx, IoExitInfo.n.u16Port, cbValue);
7842	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
7843	{
7844	/* Raise #DB. */
7845	pVmcb->guest.u64DR6 = pCtx->dr[6];
7846	pVmcb->guest.u64DR7 = pCtx->dr[7];
7847	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
7848	hmR0SvmSetPendingXcptDB(pVCpu);
7849	}
7850	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
7851	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
7852	else if ( rcStrict2 != VINF_SUCCESS
7853	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
7854	rcStrict = rcStrict2;
7855	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
7856
7857	HM_RESTORE_PREEMPT();
7858	VMMRZCallRing3Enable(pVCpu);
7859	}
7860
7861	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7862	}
7863	#ifdef VBOX_STRICT
7864	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
7865	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
7866	Assert(IoExitInfo.n.u1Type == SVM_IOIO_READ);
7867	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
7868	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
7869	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
7870	Assert(IoExitInfo.n.u1Type == SVM_IOIO_WRITE);
7871	else
7872	{
7873	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
7874	* statuses, that the VMM device and some others may return. See
7875	* IOM_SUCCESS() for guidance. */
7876	AssertMsg( RT_FAILURE(rcStrict)
7877	\|\| rcStrict == VINF_SUCCESS
7878	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
7879	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
7880	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
7881	\|\| rcStrict == VINF_EM_DBG_STEPPED
7882	\|\| rcStrict == VINF_EM_RAW_TO_R3
7883	\|\| rcStrict == VINF_EM_TRIPLE_FAULT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7884	}
7885	#endif
7886	}
7887	else
7888	{
7889	/*
7890	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
7891	*/
7892	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7893	STAM_COUNTER_INC(!IoExitInfo.n.u1Str
7894	? IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
7895	: IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
7896	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
7897	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, IoExitInfo.n.u1Rep ? "REP " : "",
7898	IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? "OUT" : "IN", IoExitInfo.n.u1Str ? "S" : "", IoExitInfo.n.u16Port, uIOWidth));
7899
7900	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
7901	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7902
7903	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
7904	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
7905	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
7906	}
7907	return rcStrict;
7908	}
7909
7910
7911	/**
7912	* \#VMEXIT handler for Nested Page-faults (SVM_EXIT_NPF). Conditional \#VMEXIT.
7913	*/
7914	HMSVM_EXIT_DECL hmR0SvmExitNestedPF(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7915	{
7916	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7917	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7918	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
7919
7920	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7921	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7922	Assert(pVM->hmr0.s.fNestedPaging);
7923
7924	/* See AMD spec. 15.25.6 "Nested versus Guest Page Faults, Fault Ordering" for VMCB details for #NPF. */
7925	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7926	RTGCPHYS GCPhysFaultAddr = pVmcb->ctrl.u64ExitInfo2;
7927	uint32_t u32ErrCode = pVmcb->ctrl.u64ExitInfo1; /* Note! High bits in EXITINFO1 may contain additional info and are
7928	thus intentionally not copied into u32ErrCode. */
7929
7930	Log4Func(("#NPF at CS:RIP=%04x:%RX64 GCPhysFaultAddr=%RGp ErrCode=%#x cbInstrFetched=%u %.15Rhxs\n", pCtx->cs.Sel, pCtx->rip, GCPhysFaultAddr,
7931	u32ErrCode, pVmcb->ctrl.cbInstrFetched, pVmcb->ctrl.abInstr));
7932
7933	/*
7934	* TPR patching for 32-bit guests, using the reserved bit in the page tables for MMIO regions.
7935	*/
7936	if ( pVM->hm.s.fTprPatchingAllowed
7937	&& (GCPhysFaultAddr & GUEST_PAGE_OFFSET_MASK) == XAPIC_OFF_TPR
7938	&& ( !(u32ErrCode & X86_TRAP_PF_P) /* Not present */
7939	\|\| (u32ErrCode & (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) == (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) /* MMIO page. */
7940	&& !CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
7941	&& !CPUMIsGuestInLongModeEx(pCtx)
7942	&& !CPUMGetGuestCPL(pVCpu)
7943	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
7944	{
7945	RTGCPHYS GCPhysApicBase = APICGetBaseMsrNoCheck(pVCpu);
7946	GCPhysApicBase &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
7947
7948	if (GCPhysFaultAddr == GCPhysApicBase + XAPIC_OFF_TPR)
7949	{
7950	/* Only attempt to patch the instruction once. */
7951	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
7952	if (!pPatch)
7953	return VINF_EM_HM_PATCH_TPR_INSTR;
7954	}
7955	}
7956
7957	/*
7958	* Determine the nested paging mode.
7959	*/
7960	/** @todo r=bird: Gotta love this nested paging hacking we're still carrying with us... (Split PGM_TYPE_NESTED.) */
7961	PGMMODE const enmNestedPagingMode = PGMGetHostMode(pVM);
7962
7963	/*
7964	* MMIO optimization using the reserved (RSVD) bit in the guest page tables for MMIO pages.
7965	*/
7966	Assert((u32ErrCode & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) != X86_TRAP_PF_RSVD);
7967	if ((u32ErrCode & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
7968	{
7969	/*
7970	* If event delivery causes an MMIO #NPF, go back to instruction emulation as otherwise
7971	* injecting the original pending event would most likely cause the same MMIO #NPF.
7972	*/
7973	if (pVCpu->hm.s.Event.fPending)
7974	{
7975	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
7976	return VINF_EM_RAW_INJECT_TRPM_EVENT;
7977	}
7978
7979	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
7980	VBOXSTRICTRC rcStrict;
7981	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
7982	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
7983	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
7984	if (!pExitRec)
7985	{
7986
7987	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, enmNestedPagingMode, CPUMCTX2CORE(pCtx), GCPhysFaultAddr,
7988	u32ErrCode);
7989
7990	/*
7991	* If we succeed, resume guest execution.
7992	*
7993	* If we fail in interpreting the instruction because we couldn't get the guest
7994	* physical address of the page containing the instruction via the guest's page
7995	* tables (we would invalidate the guest page in the host TLB), resume execution
7996	* which would cause a guest page fault to let the guest handle this weird case.
7997	*
7998	* See @bugref{6043}.
7999	*/
8000	if ( rcStrict == VINF_SUCCESS
8001	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
8002	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
8003	{
8004	/* Successfully handled MMIO operation. */
8005	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8006	rcStrict = VINF_SUCCESS;
8007	}
8008	}
8009	else
8010	{
8011	/*
8012	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
8013	*/
8014	Assert(pCtx == &pVCpu->cpum.GstCtx);
8015	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8016	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
8017	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhysFaultAddr));
8018
8019	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
8020	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8021
8022	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
8023	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
8024	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
8025	}
8026	return rcStrict;
8027	}
8028
8029	/*
8030	* Nested page-fault.
8031	*/
8032	TRPMAssertXcptPF(pVCpu, GCPhysFaultAddr, u32ErrCode);
8033	int rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, enmNestedPagingMode, u32ErrCode, CPUMCTX2CORE(pCtx), GCPhysFaultAddr);
8034	TRPMResetTrap(pVCpu);
8035
8036	Log4Func(("#NPF: PGMR0Trap0eHandlerNestedPaging returns %Rrc CS:RIP=%04x:%RX64\n", rc, pCtx->cs.Sel, pCtx->rip));
8037
8038	/*
8039	* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}.
8040	*/
8041	if ( rc == VINF_SUCCESS
8042	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8043	\|\| rc == VERR_PAGE_NOT_PRESENT)
8044	{
8045	/* We've successfully synced our shadow page tables. */
8046	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8047	rc = VINF_SUCCESS;
8048	}
8049
8050	/*
8051	* If delivering an event causes an #NPF (and not MMIO), we shall resolve the fault and
8052	* re-inject the original event.
8053	*/
8054	if (pVCpu->hm.s.Event.fPending)
8055	{
8056	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflectNPF);
8057
8058	/*
8059	* If the #NPF handler requested emulation of the instruction, ignore it.
8060	* We need to re-inject the original event so as to not lose it.
8061	* Reproducible when booting ReactOS 0.4.12 with BTRFS (installed using BootCD,
8062	* LiveCD is broken for other reasons).
8063	*/
8064	if (rc == VINF_EM_RAW_EMULATE_INSTR)
8065	rc = VINF_EM_RAW_INJECT_TRPM_EVENT;
8066	}
8067
8068	return rc;
8069	}
8070
8071
8072	/**
8073	* \#VMEXIT handler for virtual interrupt (SVM_EXIT_VINTR). Conditional
8074	* \#VMEXIT.
8075	*/
8076	HMSVM_EXIT_DECL hmR0SvmExitVIntr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8077	{
8078	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8079	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
8080
8081	/* Indicate that we no longer need to #VMEXIT when the guest is ready to receive NMIs, it is now ready. */
8082	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8083	hmR0SvmClearIntWindowExiting(pVCpu, pVmcb);
8084
8085	/* Deliver the pending interrupt via hmR0SvmEvaluatePendingEvent() and resume guest execution. */
8086	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
8087	return VINF_SUCCESS;
8088	}
8089
8090
8091	/**
8092	* \#VMEXIT handler for task switches (SVM_EXIT_TASK_SWITCH). Conditional
8093	* \#VMEXIT.
8094	*/
8095	HMSVM_EXIT_DECL hmR0SvmExitTaskSwitch(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8096	{
8097	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8098	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8099
8100	#ifndef HMSVM_ALWAYS_TRAP_TASK_SWITCH
8101	Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
8102	#endif
8103
8104	/* Check if this task-switch occurred while delivering an event through the guest IDT. */
8105	if (pVCpu->hm.s.Event.fPending) /* Can happen with exceptions/NMI. See @bugref{8411}. */
8106	{
8107	/*
8108	* AMD-V provides us with the exception which caused the TS; we collect
8109	* the information in the call to hmR0SvmCheckExitDueToEventDelivery().
8110	*/
8111	Log4Func(("TS occurred during event delivery\n"));
8112	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8113	return VINF_EM_RAW_INJECT_TRPM_EVENT;
8114	}
8115
8116	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8117	* emulation. */
8118	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8119	return VERR_EM_INTERPRETER;
8120	}
8121
8122
8123	/**
8124	* \#VMEXIT handler for VMMCALL (SVM_EXIT_VMMCALL). Conditional \#VMEXIT.
8125	*/
8126	HMSVM_EXIT_DECL hmR0SvmExitVmmCall(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8127	{
8128	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8129	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8130
8131	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8132	if (pVM->hm.s.fTprPatchingAllowed)
8133	{
8134	int rc = hmEmulateSvmMovTpr(pVM, pVCpu);
8135	if (rc != VERR_NOT_FOUND)
8136	{
8137	Log4Func(("hmEmulateSvmMovTpr returns %Rrc\n", rc));
8138	return rc;
8139	}
8140	}
8141
8142	if (EMAreHypercallInstructionsEnabled(pVCpu))
8143	{
8144	unsigned cbInstr;
8145	if (hmR0SvmSupportsNextRipSave(pVCpu))
8146	{
8147	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8148	cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8149	}
8150	else
8151	{
8152	PDISCPUSTATE pDis = &pVCpu->hmr0.s.svm.DisState;
8153	int rc = EMInterpretDisasCurrent(pVCpu->CTX_SUFF(pVM), pVCpu, pDis, &cbInstr);
8154	if ( rc == VINF_SUCCESS
8155	&& pDis->pCurInstr->uOpcode == OP_VMMCALL)
8156	Assert(cbInstr > 0);
8157	else
8158	cbInstr = 0;
8159	}
8160
8161	VBOXSTRICTRC rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
8162	if (RT_SUCCESS(rcStrict))
8163	{
8164	/* Only update the RIP if we're continuing guest execution and not in the case
8165	of say VINF_GIM_R3_HYPERCALL. */
8166	if (rcStrict == VINF_SUCCESS)
8167	hmR0SvmAdvanceRip(pVCpu, cbInstr);
8168
8169	return VBOXSTRICTRC_VAL(rcStrict);
8170	}
8171	else
8172	Log4Func(("GIMHypercall returns %Rrc -> #UD\n", VBOXSTRICTRC_VAL(rcStrict)));
8173	}
8174
8175	hmR0SvmSetPendingXcptUD(pVCpu);
8176	return VINF_SUCCESS;
8177	}
8178
8179
8180	/**
8181	* \#VMEXIT handler for VMMCALL (SVM_EXIT_VMMCALL). Conditional \#VMEXIT.
8182	*/
8183	HMSVM_EXIT_DECL hmR0SvmExitPause(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8184	{
8185	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8186
8187	unsigned cbInstr;
8188	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8189	if (fSupportsNextRipSave)
8190	{
8191	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8192	cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8193	}
8194	else
8195	{
8196	PDISCPUSTATE pDis = &pVCpu->hmr0.s.svm.DisState;
8197	int rc = EMInterpretDisasCurrent(pVCpu->CTX_SUFF(pVM), pVCpu, pDis, &cbInstr);
8198	if ( rc == VINF_SUCCESS
8199	&& pDis->pCurInstr->uOpcode == OP_PAUSE)
8200	Assert(cbInstr > 0);
8201	else
8202	cbInstr = 0;
8203	}
8204
8205	/** @todo The guest has likely hit a contended spinlock. We might want to
8206	* poke a schedule different guest VCPU. */
8207	hmR0SvmAdvanceRip(pVCpu, cbInstr);
8208	return VINF_EM_RAW_INTERRUPT;
8209	}
8210
8211
8212	/**
8213	* \#VMEXIT handler for FERR intercept (SVM_EXIT_FERR_FREEZE). Conditional
8214	* \#VMEXIT.
8215	*/
8216	HMSVM_EXIT_DECL hmR0SvmExitFerrFreeze(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8217	{
8218	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8219	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0);
8220	Assert(!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE));
8221
8222	Log4Func(("Raising IRQ 13 in response to #FERR\n"));
8223	return PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13 /* u8Irq /, 1 / u8Level /, 0 / uTagSrc */);
8224	}
8225
8226
8227	/**
8228	* \#VMEXIT handler for IRET (SVM_EXIT_IRET). Conditional \#VMEXIT.
8229	*/
8230	HMSVM_EXIT_DECL hmR0SvmExitIret(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8231	{
8232	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8233
8234	/* Indicate that we no longer need to #VMEXIT when the guest is ready to receive NMIs, it is now (almost) ready. */
8235	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8236	hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_IRET);
8237
8238	/* Emulate the IRET. We have to execute the IRET before an NMI, but must potentially
8239	* deliver a pending NMI right after. If the IRET faults, an NMI can come before the
8240	* handler executes. Yes, x86 is ugly.
8241	*/
8242	return VINF_EM_RAW_EMULATE_INSTR;
8243	}
8244
8245
8246	/**
8247	* \#VMEXIT handler for page-fault exceptions (SVM_EXIT_XCPT_14).
8248	* Conditional \#VMEXIT.
8249	*/
8250	HMSVM_EXIT_DECL hmR0SvmExitXcptPF(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8251	{
8252	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8253	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8254	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8255
8256	/* See AMD spec. 15.12.15 "#PF (Page Fault)". */
8257	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8258	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8259	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8260	uint32_t uErrCode = pVmcb->ctrl.u64ExitInfo1;
8261	uint64_t const uFaultAddress = pVmcb->ctrl.u64ExitInfo2;
8262
8263	#if defined(HMSVM_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMSVM_ALWAYS_TRAP_PF)
8264	if (pVM->hmr0.s.fNestedPaging)
8265	{
8266	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
8267	if ( !pSvmTransient->fVectoringDoublePF
8268	\|\| CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
8269	{
8270	/* A genuine guest #PF, reflect it to the guest. */
8271	hmR0SvmSetPendingXcptPF(pVCpu, uErrCode, uFaultAddress);
8272	Log4Func(("#PF: Guest page fault at %04X:%RGv FaultAddr=%RX64 ErrCode=%#x\n", pCtx->cs.Sel, (RTGCPTR)pCtx->rip,
8273	uFaultAddress, uErrCode));
8274	}
8275	else
8276	{
8277	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8278	hmR0SvmSetPendingXcptDF(pVCpu);
8279	Log4Func(("Pending #DF due to vectoring #PF. NP\n"));
8280	}
8281	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8282	return VINF_SUCCESS;
8283	}
8284	#endif
8285
8286	Assert(!pVM->hmr0.s.fNestedPaging);
8287
8288	/*
8289	* TPR patching shortcut for APIC TPR reads and writes; only applicable to 32-bit guests.
8290	*/
8291	if ( pVM->hm.s.fTprPatchingAllowed
8292	&& (uFaultAddress & 0xfff) == XAPIC_OFF_TPR
8293	&& !(uErrCode & X86_TRAP_PF_P) /* Not present. */
8294	&& !CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
8295	&& !CPUMIsGuestInLongModeEx(pCtx)
8296	&& !CPUMGetGuestCPL(pVCpu)
8297	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8298	{
8299	RTGCPHYS GCPhysApicBase;
8300	GCPhysApicBase = APICGetBaseMsrNoCheck(pVCpu);
8301	GCPhysApicBase &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
8302
8303	/* Check if the page at the fault-address is the APIC base. */
8304	PGMPTWALK Walk;
8305	int rc2 = PGMGstGetPage(pVCpu, (RTGCPTR)uFaultAddress, &Walk);
8306	if ( rc2 == VINF_SUCCESS
8307	&& Walk.GCPhys == GCPhysApicBase)
8308	{
8309	/* Only attempt to patch the instruction once. */
8310	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
8311	if (!pPatch)
8312	return VINF_EM_HM_PATCH_TPR_INSTR;
8313	}
8314	}
8315
8316	Log4Func(("#PF: uFaultAddress=%#RX64 CS:RIP=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", uFaultAddress, pCtx->cs.Sel,
8317	pCtx->rip, uErrCode, pCtx->cr3));
8318
8319	/*
8320	* If it's a vectoring #PF, emulate injecting the original event injection as
8321	* PGMTrap0eHandler() is incapable of differentiating between instruction emulation and
8322	* event injection that caused a #PF. See @bugref{6607}.
8323	*/
8324	if (pSvmTransient->fVectoringPF)
8325	{
8326	Assert(pVCpu->hm.s.Event.fPending);
8327	return VINF_EM_RAW_INJECT_TRPM_EVENT;
8328	}
8329
8330	TRPMAssertXcptPF(pVCpu, uFaultAddress, uErrCode);
8331	int rc = PGMTrap0eHandler(pVCpu, uErrCode, CPUMCTX2CORE(pCtx), (RTGCPTR)uFaultAddress);
8332
8333	Log4Func(("#PF: rc=%Rrc\n", rc));
8334
8335	if (rc == VINF_SUCCESS)
8336	{
8337	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
8338	TRPMResetTrap(pVCpu);
8339	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8340	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8341	return rc;
8342	}
8343
8344	if (rc == VINF_EM_RAW_GUEST_TRAP)
8345	{
8346	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
8347
8348	/*
8349	* If a nested-guest delivers a #PF and that causes a #PF which is -not- a shadow #PF,
8350	* we should simply forward the #PF to the guest and is up to the nested-hypervisor to
8351	* determine whether it is a nested-shadow #PF or a #DF, see @bugref{7243#c121}.
8352	*/
8353	if ( !pSvmTransient->fVectoringDoublePF
8354	\|\| CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
8355	{
8356	/* It's a guest (or nested-guest) page fault and needs to be reflected. */
8357	uErrCode = TRPMGetErrorCode(pVCpu); /* The error code might have been changed. */
8358	TRPMResetTrap(pVCpu);
8359
8360	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
8361	/* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
8362	if ( CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
8363	&& CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_PF))
8364	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_XCPT_PF, uErrCode, uFaultAddress);
8365	#endif
8366
8367	hmR0SvmSetPendingXcptPF(pVCpu, uErrCode, uFaultAddress);
8368	}
8369	else
8370	{
8371	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8372	TRPMResetTrap(pVCpu);
8373	hmR0SvmSetPendingXcptDF(pVCpu);
8374	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
8375	}
8376
8377	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8378	return VINF_SUCCESS;
8379	}
8380
8381	TRPMResetTrap(pVCpu);
8382	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
8383	return rc;
8384	}
8385
8386
8387
8388	/**
8389	* \#VMEXIT handler for division overflow exceptions (SVM_EXIT_XCPT_1).
8390	* Conditional \#VMEXIT.
8391	*/
8392	HMSVM_EXIT_DECL hmR0SvmExitXcptDE(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8393	{
8394	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8395	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
8396	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
8397
8398	/* Paranoia; Ensure we cannot be called as a result of event delivery. */
8399	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8400	Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
8401
8402	int rc = VERR_SVM_UNEXPECTED_XCPT_EXIT;
8403	if (pVCpu->hm.s.fGCMTrapXcptDE)
8404	{
8405	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8406	uint8_t cbInstr = 0;
8407	VBOXSTRICTRC rcStrict = GCMXcptDE(pVCpu, &pVCpu->cpum.GstCtx, NULL /* pDis */, &cbInstr);
8408	if (rcStrict == VINF_SUCCESS)
8409	rc = VINF_SUCCESS; /* Restart instruction with modified guest register context. */
8410	else if (rcStrict == VERR_NOT_FOUND)
8411	rc = VERR_NOT_FOUND; /* Deliver the exception. */
8412	else
8413	Assert(RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
8414	}
8415
8416	/* If the GCM #DE exception handler didn't succeed or wasn't needed, raise #DE. */
8417	if (RT_FAILURE(rc))
8418	{
8419	hmR0SvmSetPendingXcptDE(pVCpu);
8420	rc = VINF_SUCCESS;
8421	}
8422
8423	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
8424	return rc;
8425	}
8426
8427
8428	/**
8429	* \#VMEXIT handler for undefined opcode (SVM_EXIT_XCPT_6).
8430	* Conditional \#VMEXIT.
8431	*/
8432	HMSVM_EXIT_DECL hmR0SvmExitXcptUD(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8433	{
8434	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8435	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
8436	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8437
8438	/* Paranoia; Ensure we cannot be called as a result of event delivery. */
8439	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
8440	Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
8441
8442	/** @todo if we accumulate more optional stuff here, we ought to combine the
8443	* reading of opcode bytes to avoid doing more than once. */
8444
8445	VBOXSTRICTRC rcStrict = VERR_SVM_UNEXPECTED_XCPT_EXIT;
8446	if (pVCpu->hm.s.fGIMTrapXcptUD)
8447	{
8448	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8449	uint8_t cbInstr = 0;
8450	rcStrict = GIMXcptUD(pVCpu, &pVCpu->cpum.GstCtx, NULL /* pDis */, &cbInstr);
8451	if (rcStrict == VINF_SUCCESS)
8452	{
8453	/* #UD #VMEXIT does not have valid NRIP information, manually advance RIP. See @bugref{7270#c170}. */
8454	hmR0SvmAdvanceRip(pVCpu, cbInstr);
8455	rcStrict = VINF_SUCCESS;
8456	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
8457	}
8458	else if (rcStrict == VINF_GIM_HYPERCALL_CONTINUING)
8459	rcStrict = VINF_SUCCESS;
8460	else if (rcStrict == VINF_GIM_R3_HYPERCALL)
8461	rcStrict = VINF_GIM_R3_HYPERCALL;
8462	else
8463	{
8464	Assert(RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
8465	rcStrict = VERR_SVM_UNEXPECTED_XCPT_EXIT;
8466	}
8467	}
8468
8469	if (pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit)
8470	{
8471	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS
8472	\| CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_EFER);
8473	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
8474	{
8475	/* Ideally, IEM should just handle all these special #UD situations, but
8476	we don't quite trust things to behave optimially when doing that. So,
8477	for now we'll restrict ourselves to a handful of possible sysenter and
8478	sysexit encodings that we filter right here. */
8479	uint8_t abInstr[SVM_CTRL_GUEST_INSTR_BYTES_MAX];
8480	uint8_t cbInstr = pVmcb->ctrl.cbInstrFetched;
8481	uint32_t const uCpl = CPUMGetGuestCPL(pVCpu);
8482	uint8_t const cbMin = uCpl != 0 ? 2 : 1 + 2;
8483	RTGCPTR const GCPtrInstr = pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base;
8484	if (cbInstr < cbMin \|\| cbInstr > SVM_CTRL_GUEST_INSTR_BYTES_MAX)
8485	{
8486	cbInstr = cbMin;
8487	int rc2 = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, GCPtrInstr, cbInstr);
8488	AssertRCStmt(rc2, cbInstr = 0);
8489	}
8490	else
8491	memcpy(abInstr, pVmcb->ctrl.abInstr, cbInstr); /* unlikely */
8492	if ( cbInstr == 0 /* read error */
8493	\|\| (cbInstr >= 2 && abInstr[0] == 0x0f && abInstr[1] == 0x34) /* sysenter */
8494	\|\| ( uCpl == 0
8495	&& ( ( cbInstr >= 2 && abInstr[0] == 0x0f && abInstr[1] == 0x35) /* sysexit */
8496	\|\| ( cbInstr >= 3 && abInstr[1] == 0x0f && abInstr[2] == 0x35 /* rex.w sysexit */
8497	&& (abInstr[0] & (X86_OP_REX_W \| 0xf0)) == X86_OP_REX_W))))
8498	{
8499	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
8500	\| CPUMCTX_EXTRN_SREG_MASK /* without ES+DS+GS the app will #GP later - go figure */);
8501	Log6(("hmR0SvmExitXcptUD: sysenter/sysexit: %.*Rhxs at %#llx CPL=%u\n", cbInstr, abInstr, GCPtrInstr, uCpl));
8502	rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), GCPtrInstr, abInstr, cbInstr);
8503	Log6(("hmR0SvmExitXcptUD: sysenter/sysexit: rcStrict=%Rrc %04x:%08RX64 %08RX64 %04x:%08RX64\n",
8504	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags.u,
8505	pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp));
8506	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8507	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); /** @todo Lazy bird. */
8508	if (rcStrict == VINF_IEM_RAISED_XCPT)
8509	rcStrict = VINF_SUCCESS;
8510	return rcStrict;
8511	}
8512	Log6(("hmR0SvmExitXcptUD: not sysenter/sysexit: %.*Rhxs at %#llx CPL=%u\n", cbInstr, abInstr, GCPtrInstr, uCpl));
8513	}
8514	else
8515	Log6(("hmR0SvmExitXcptUD: not in long mode at %04x:%llx\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
8516	}
8517
8518	/* If the GIM #UD exception handler didn't succeed for some reason or wasn't needed, raise #UD. */
8519	if (RT_FAILURE(rcStrict))
8520	{
8521	hmR0SvmSetPendingXcptUD(pVCpu);
8522	rcStrict = VINF_SUCCESS;
8523	}
8524
8525	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8526	return rcStrict;
8527	}
8528
8529
8530	/**
8531	* \#VMEXIT handler for math-fault exceptions (SVM_EXIT_XCPT_16).
8532	* Conditional \#VMEXIT.
8533	*/
8534	HMSVM_EXIT_DECL hmR0SvmExitXcptMF(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8535	{
8536	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8537	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8538	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8539
8540	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8541	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8542
8543	/* Paranoia; Ensure we cannot be called as a result of event delivery. */
8544	Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
8545
8546	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8547
8548	if (!(pCtx->cr0 & X86_CR0_NE))
8549	{
8550	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8551	PDISSTATE pDis = &pVCpu->hmr0.s.svm.DisState;
8552	unsigned cbInstr;
8553	int rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbInstr);
8554	if (RT_SUCCESS(rc))
8555	{
8556	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
8557	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13 /* u8Irq /, 1 / u8Level /, 0 / uTagSrc */);
8558	if (RT_SUCCESS(rc))
8559	hmR0SvmAdvanceRip(pVCpu, cbInstr);
8560	}
8561	else
8562	Log4Func(("EMInterpretDisasCurrent returned %Rrc uOpCode=%#x\n", rc, pDis->pCurInstr->uOpcode));
8563	return rc;
8564	}
8565
8566	hmR0SvmSetPendingXcptMF(pVCpu);
8567	return VINF_SUCCESS;
8568	}
8569
8570
8571	/**
8572	* \#VMEXIT handler for debug exceptions (SVM_EXIT_XCPT_1). Conditional
8573	* \#VMEXIT.
8574	*/
8575	HMSVM_EXIT_DECL hmR0SvmExitXcptDB(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8576	{
8577	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8578	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8579	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8580	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8581
8582	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
8583	{
8584	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
8585	return VINF_EM_RAW_INJECT_TRPM_EVENT;
8586	}
8587
8588	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8589
8590	/*
8591	* This can be a fault-type #DB (instruction breakpoint) or a trap-type #DB (data
8592	* breakpoint). However, for both cases DR6 and DR7 are updated to what the exception
8593	* handler expects. See AMD spec. 15.12.2 "#DB (Debug)".
8594	*/
8595	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8596	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
8597	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8598	int rc = DBGFTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), pVmcb->guest.u64DR6, pVCpu->hm.s.fSingleInstruction);
8599	if (rc == VINF_EM_RAW_GUEST_TRAP)
8600	{
8601	Log5(("hmR0SvmExitXcptDB: DR6=%#RX64 -> guest trap\n", pVmcb->guest.u64DR6));
8602	if (CPUMIsHyperDebugStateActive(pVCpu))
8603	CPUMSetGuestDR6(pVCpu, CPUMGetGuestDR6(pVCpu) \| pVmcb->guest.u64DR6);
8604
8605	/* Reflect the exception back to the guest. */
8606	hmR0SvmSetPendingXcptDB(pVCpu);
8607	rc = VINF_SUCCESS;
8608	}
8609
8610	/*
8611	* Update DR6.
8612	*/
8613	if (CPUMIsHyperDebugStateActive(pVCpu))
8614	{
8615	Log5(("hmR0SvmExitXcptDB: DR6=%#RX64 -> %Rrc\n", pVmcb->guest.u64DR6, rc));
8616	pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
8617	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
8618	}
8619	else
8620	{
8621	AssertMsg(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc));
8622	Assert(!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu));
8623	}
8624
8625	return rc;
8626	}
8627
8628
8629	/**
8630	* \#VMEXIT handler for alignment check exceptions (SVM_EXIT_XCPT_17).
8631	* Conditional \#VMEXIT.
8632	*/
8633	HMSVM_EXIT_DECL hmR0SvmExitXcptAC(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8634	{
8635	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8636	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8637	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC);
8638
8639	SVMEVENT Event;
8640	Event.u = 0;
8641	Event.n.u1Valid = 1;
8642	Event.n.u3Type = SVM_EVENT_EXCEPTION;
8643	Event.n.u8Vector = X86_XCPT_AC;
8644	Event.n.u1ErrorCodeValid = 1;
8645	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8646	return VINF_SUCCESS;
8647	}
8648
8649
8650	/**
8651	* \#VMEXIT handler for breakpoint exceptions (SVM_EXIT_XCPT_3).
8652	* Conditional \#VMEXIT.
8653	*/
8654	HMSVM_EXIT_DECL hmR0SvmExitXcptBP(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8655	{
8656	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8657	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8658	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8659	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8660
8661	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8662	VBOXSTRICTRC rc = DBGFTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
8663	if (rc == VINF_EM_RAW_GUEST_TRAP)
8664	{
8665	SVMEVENT Event;
8666	Event.u = 0;
8667	Event.n.u1Valid = 1;
8668	Event.n.u3Type = SVM_EVENT_EXCEPTION;
8669	Event.n.u8Vector = X86_XCPT_BP;
8670	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8671	rc = VINF_SUCCESS;
8672	}
8673
8674	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_DBG_BREAKPOINT);
8675	return rc;
8676	}
8677
8678
8679	/**
8680	* Hacks its way around the lovely mesa driver's backdoor accesses.
8681	*
8682	* @sa hmR0VmxHandleMesaDrvGp
8683	*/
8684	static int hmR0SvmHandleMesaDrvGp(PVMCPUCC pVCpu, PCPUMCTX pCtx, PCSVMVMCB pVmcb)
8685	{
8686	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_GPRS_MASK);
8687	Log(("hmR0SvmHandleMesaDrvGp: at %04x:%08RX64 rcx=%RX64 rbx=%RX64\n",
8688	pVmcb->guest.CS.u16Sel, pVmcb->guest.u64RIP, pCtx->rcx, pCtx->rbx));
8689	RT_NOREF(pCtx, pVmcb);
8690
8691	/* For now we'll just skip the instruction. */
8692	hmR0SvmAdvanceRip(pVCpu, 1);
8693	return VINF_SUCCESS;
8694	}
8695
8696
8697	/**
8698	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
8699	* backdoor logging w/o checking what it is running inside.
8700	*
8701	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
8702	* backdoor port and magic numbers loaded in registers.
8703	*
8704	* @returns true if it is, false if it isn't.
8705	* @sa hmR0VmxIsMesaDrvGp
8706	*/
8707	DECLINLINE(bool) hmR0SvmIsMesaDrvGp(PVMCPUCC pVCpu, PCPUMCTX pCtx, PCSVMVMCB pVmcb)
8708	{
8709	/* Check magic and port. */
8710	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
8711	/Log8(("hmR0SvmIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->fExtrn & CPUMCTX_EXTRN_RAX ? pVmcb->guest.u64RAX : pCtx->rax, pCtx->rdx));/
8712	if (pCtx->dx != UINT32_C(0x5658))
8713	return false;
8714	if ((pCtx->fExtrn & CPUMCTX_EXTRN_RAX ? pVmcb->guest.u64RAX : pCtx->rax) != UINT32_C(0x564d5868))
8715	return false;
8716
8717	/* Check that it is #GP(0). */
8718	if (pVmcb->ctrl.u64ExitInfo1 != 0)
8719	return false;
8720
8721	/* Flat ring-3 CS. */
8722	/Log8(("hmR0SvmIsMesaDrvGp: u8CPL=%d base=%RX64\n", pVmcb->guest.u8CPL, pCtx->fExtrn & CPUMCTX_EXTRN_CS ? pVmcb->guest.CS.u64Base : pCtx->cs.u64Base));/
8723	if (pVmcb->guest.u8CPL != 3)
8724	return false;
8725	if ((pCtx->fExtrn & CPUMCTX_EXTRN_CS ? pVmcb->guest.CS.u64Base : pCtx->cs.u64Base) != 0)
8726	return false;
8727
8728	/* 0xed: IN eAX,dx */
8729	if (pVmcb->ctrl.cbInstrFetched < 1) /* unlikely, it turns out. */
8730	{
8731	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_GPRS_MASK
8732	\| CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_EFER);
8733	uint8_t abInstr[1];
8734	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
8735	/Log8(("hmR0SvmIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0])); /
8736	if (RT_FAILURE(rc))
8737	return false;
8738	if (abInstr[0] != 0xed)
8739	return false;
8740	}
8741	else
8742	{
8743	/Log8(("hmR0SvmIsMesaDrvGp: %#x\n", pVmcb->ctrl.abInstr));/
8744	if (pVmcb->ctrl.abInstr[0] != 0xed)
8745	return false;
8746	}
8747	return true;
8748	}
8749
8750
8751	/**
8752	* \#VMEXIT handler for general protection faults (SVM_EXIT_XCPT_BP).
8753	* Conditional \#VMEXIT.
8754	*/
8755	HMSVM_EXIT_DECL hmR0SvmExitXcptGP(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8756	{
8757	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8758	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8759	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8760
8761	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8762	Assert(pSvmTransient->u64ExitCode == pVmcb->ctrl.u64ExitCode);
8763
8764	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8765	if ( !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
8766	\|\| !hmR0SvmIsMesaDrvGp(pVCpu, pCtx, pVmcb))
8767	{
8768	SVMEVENT Event;
8769	Event.u = 0;
8770	Event.n.u1Valid = 1;
8771	Event.n.u3Type = SVM_EVENT_EXCEPTION;
8772	Event.n.u8Vector = X86_XCPT_GP;
8773	Event.n.u1ErrorCodeValid = 1;
8774	Event.n.u32ErrorCode = (uint32_t)pVmcb->ctrl.u64ExitInfo1;
8775	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8776	return VINF_SUCCESS;
8777	}
8778	return hmR0SvmHandleMesaDrvGp(pVCpu, pCtx, pVmcb);
8779	}
8780
8781
8782	/**
8783	* \#VMEXIT handler for generic exceptions. Conditional \#VMEXIT.
8784	*/
8785	HMSVM_EXIT_DECL hmR0SvmExitXcptGeneric(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8786	{
8787	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8788	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8789
8790	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8791	uint8_t const uVector = pVmcb->ctrl.u64ExitCode - SVM_EXIT_XCPT_0;
8792	uint32_t const uErrCode = pVmcb->ctrl.u64ExitInfo1;
8793	Assert(pSvmTransient->u64ExitCode == pVmcb->ctrl.u64ExitCode);
8794	Assert(uVector <= X86_XCPT_LAST);
8795	Log4Func(("uVector=%#x uErrCode=%u\n", uVector, uErrCode));
8796
8797	SVMEVENT Event;
8798	Event.u = 0;
8799	Event.n.u1Valid = 1;
8800	Event.n.u3Type = SVM_EVENT_EXCEPTION;
8801	Event.n.u8Vector = uVector;
8802	switch (uVector)
8803	{
8804	/* Shouldn't be here for reflecting #PFs (among other things, the fault address isn't passed along). */
8805	case X86_XCPT_PF: AssertMsgFailed(("hmR0SvmExitXcptGeneric: Unexpected exception")); return VERR_SVM_IPE_5;
8806	case X86_XCPT_DF:
8807	case X86_XCPT_TS:
8808	case X86_XCPT_NP:
8809	case X86_XCPT_SS:
8810	case X86_XCPT_GP:
8811	case X86_XCPT_AC:
8812	{
8813	Event.n.u1ErrorCodeValid = 1;
8814	Event.n.u32ErrorCode = uErrCode;
8815	break;
8816	}
8817	}
8818
8819	#ifdef VBOX_WITH_STATISTICS
8820	switch (uVector)
8821	{
8822	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE); break;
8823	case X86_XCPT_DB: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB); break;
8824	case X86_XCPT_BP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP); break;
8825	case X86_XCPT_OF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
8826	case X86_XCPT_BR: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBR); break;
8827	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD); break;
8828	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
8829	case X86_XCPT_DF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDF); break;
8830	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS); break;
8831	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP); break;
8832	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS); break;
8833	case X86_XCPT_GP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP); break;
8834	case X86_XCPT_PF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF); break;
8835	case X86_XCPT_MF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF); break;
8836	case X86_XCPT_AC: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC); break;
8837	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF); break;
8838	default:
8839	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
8840	break;
8841	}
8842	#endif
8843
8844	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8845	return VINF_SUCCESS;
8846	}
8847
8848
8849	/**
8850	* \#VMEXIT handler for software interrupt (INTn). Conditional \#VMEXIT (debug).
8851	*/
8852	HMSVM_EXIT_DECL hmR0SvmExitSwInt(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8853	{
8854	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8855	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8856
8857	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8858	SVMEVENT Event;
8859	Event.u = 0;
8860	Event.n.u1Valid = 1;
8861	Event.n.u3Type = SVM_EVENT_SOFTWARE_INT;
8862	Event.n.u8Vector = pVmcb->ctrl.u64ExitInfo1 & 0xff;
8863	Log4Func(("uVector=%#x\n", Event.n.u8Vector));
8864	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8865	return VINF_SUCCESS;
8866	}
8867
8868
8869	/**
8870	* Generic exit handler that interprets the current instruction
8871	*
8872	* Useful exit that only gets triggered by dtrace and the debugger. Caller does
8873	* the exit logging, and this function does the rest.
8874	*/
8875	static VBOXSTRICTRC hmR0SvmExitInterpretInstruction(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient,
8876	uint64_t fExtraImport, uint64_t fHmChanged)
8877	{
8878	#if 1
8879	RT_NOREF(pSvmTransient);
8880	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fExtraImport);
8881	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
8882	if (rcStrict == VINF_SUCCESS)
8883	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, fHmChanged \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RIP);
8884	else
8885	{
8886	Log4Func(("IEMExecOne -> %Rrc\n", VBOXSTRICTRC_VAL(rcStrict) ));
8887	if (rcStrict == VINF_IEM_RAISED_XCPT)
8888	{
8889	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK \| fHmChanged);
8890	rcStrict = VINF_SUCCESS;
8891	}
8892	else
8893	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, fHmChanged);
8894	}
8895	return rcStrict;
8896	#else
8897	RT_NOREF(pVCpu, pSvmTransient, fExtraImport, fHmChanged);
8898	return VINF_EM_RAW_EMULATE_INSTR;
8899	#endif
8900	}
8901
8902
8903	/**
8904	* \#VMEXIT handler for STR. Conditional \#VMEXIT (debug).
8905	*/
8906	HMSVM_EXIT_DECL hmR0SvmExitTrRead(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8907	{
8908	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8909	Log4Func(("%04x:%08RX64\n", pSvmTransient->pVmcb->guest.CS.u16Sel, pSvmTransient->pVmcb->guest.u64RIP));
8910	return hmR0SvmExitInterpretInstruction(pVCpu, pSvmTransient, CPUMCTX_EXTRN_TR, 0);
8911	}
8912
8913
8914	/**
8915	* \#VMEXIT handler for LTR. Conditional \#VMEXIT (OS/2 TLB workaround, debug).
8916	*/
8917	HMSVM_EXIT_DECL hmR0SvmExitTrWrite(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8918	{
8919	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8920
8921	/* Workaround for lack of TLB flushing in OS/2 when returning to protected
8922	mode after a real mode call (like a BIOS call). See ticketref:20625
8923	comment 14. */
8924	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8925	if (pVM->hm.s.fMissingOS2TlbFlushWorkaround)
8926	{
8927	Log4Func(("%04x:%08RX64 TLB flush\n", pSvmTransient->pVmcb->guest.CS.u16Sel, pSvmTransient->pVmcb->guest.u64RIP));
8928	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
8929	}
8930	else
8931	Log4Func(("%04x:%08RX64\n", pSvmTransient->pVmcb->guest.CS.u16Sel, pSvmTransient->pVmcb->guest.u64RIP));
8932
8933	return hmR0SvmExitInterpretInstruction(pVCpu, pSvmTransient, CPUMCTX_EXTRN_TR \| CPUMCTX_EXTRN_GDTR, HM_CHANGED_GUEST_TR);
8934	}
8935
8936
8937	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
8938	/**
8939	* \#VMEXIT handler for CLGI (SVM_EXIT_CLGI). Conditional \#VMEXIT.
8940	*/
8941	HMSVM_EXIT_DECL hmR0SvmExitClgi(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8942	{
8943	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8944
8945	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8946	Assert(pVmcb);
8947	Assert(!pVmcb->ctrl.IntCtrl.n.u1VGifEnable);
8948
8949	VBOXSTRICTRC rcStrict;
8950	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8951	uint64_t const fImport = CPUMCTX_EXTRN_HWVIRT;
8952	if (fSupportsNextRipSave)
8953	{
8954	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| fImport);
8955	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8956	rcStrict = IEMExecDecodedClgi(pVCpu, cbInstr);
8957	}
8958	else
8959	{
8960	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fImport);
8961	rcStrict = IEMExecOne(pVCpu);
8962	}
8963
8964	if (rcStrict == VINF_SUCCESS)
8965	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
8966	else if (rcStrict == VINF_IEM_RAISED_XCPT)
8967	{
8968	rcStrict = VINF_SUCCESS;
8969	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
8970	}
8971	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
8972	return rcStrict;
8973	}
8974
8975
8976	/**
8977	* \#VMEXIT handler for STGI (SVM_EXIT_STGI). Conditional \#VMEXIT.
8978	*/
8979	HMSVM_EXIT_DECL hmR0SvmExitStgi(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8980	{
8981	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8982
8983	/*
8984	* When VGIF is not used we always intercept STGI instructions. When VGIF is used,
8985	* we only intercept STGI when events are pending for GIF to become 1.
8986	*/
8987	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8988	if (pVmcb->ctrl.IntCtrl.n.u1VGifEnable)
8989	hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_STGI);
8990
8991	VBOXSTRICTRC rcStrict;
8992	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8993	uint64_t const fImport = CPUMCTX_EXTRN_HWVIRT;
8994	if (fSupportsNextRipSave)
8995	{
8996	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| fImport);
8997	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8998	rcStrict = IEMExecDecodedStgi(pVCpu, cbInstr);
8999	}
9000	else
9001	{
9002	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fImport);
9003	rcStrict = IEMExecOne(pVCpu);
9004	}
9005
9006	if (rcStrict == VINF_SUCCESS)
9007	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
9008	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9009	{
9010	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9011	rcStrict = VINF_SUCCESS;
9012	}
9013	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
9014	return rcStrict;
9015	}
9016
9017
9018	/**
9019	* \#VMEXIT handler for VMLOAD (SVM_EXIT_VMLOAD). Conditional \#VMEXIT.
9020	*/
9021	HMSVM_EXIT_DECL hmR0SvmExitVmload(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9022	{
9023	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9024
9025	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
9026	Assert(pVmcb);
9027	Assert(!pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload);
9028
9029	VBOXSTRICTRC rcStrict;
9030	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
9031	uint64_t const fImport = CPUMCTX_EXTRN_FS \| CPUMCTX_EXTRN_GS \| CPUMCTX_EXTRN_KERNEL_GS_BASE
9032	\| CPUMCTX_EXTRN_TR \| CPUMCTX_EXTRN_LDTR \| CPUMCTX_EXTRN_SYSCALL_MSRS
9033	\| CPUMCTX_EXTRN_SYSENTER_MSRS;
9034	if (fSupportsNextRipSave)
9035	{
9036	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| fImport);
9037	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
9038	rcStrict = IEMExecDecodedVmload(pVCpu, cbInstr);
9039	}
9040	else
9041	{
9042	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fImport);
9043	rcStrict = IEMExecOne(pVCpu);
9044	}
9045
9046	if (rcStrict == VINF_SUCCESS)
9047	{
9048	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS \| HM_CHANGED_GUEST_GS
9049	\| HM_CHANGED_GUEST_TR \| HM_CHANGED_GUEST_LDTR
9050	\| HM_CHANGED_GUEST_KERNEL_GS_BASE \| HM_CHANGED_GUEST_SYSCALL_MSRS
9051	\| HM_CHANGED_GUEST_SYSENTER_MSR_MASK);
9052	}
9053	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9054	{
9055	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9056	rcStrict = VINF_SUCCESS;
9057	}
9058	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
9059	return rcStrict;
9060	}
9061
9062
9063	/**
9064	* \#VMEXIT handler for VMSAVE (SVM_EXIT_VMSAVE). Conditional \#VMEXIT.
9065	*/
9066	HMSVM_EXIT_DECL hmR0SvmExitVmsave(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9067	{
9068	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9069
9070	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
9071	Assert(!pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload);
9072
9073	VBOXSTRICTRC rcStrict;
9074	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
9075	if (fSupportsNextRipSave)
9076	{
9077	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
9078	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
9079	rcStrict = IEMExecDecodedVmsave(pVCpu, cbInstr);
9080	}
9081	else
9082	{
9083	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
9084	rcStrict = IEMExecOne(pVCpu);
9085	}
9086
9087	if (rcStrict == VINF_IEM_RAISED_XCPT)
9088	{
9089	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9090	rcStrict = VINF_SUCCESS;
9091	}
9092	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
9093	return rcStrict;
9094	}
9095
9096
9097	/**
9098	* \#VMEXIT handler for INVLPGA (SVM_EXIT_INVLPGA). Conditional \#VMEXIT.
9099	*/
9100	HMSVM_EXIT_DECL hmR0SvmExitInvlpga(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9101	{
9102	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9103
9104	VBOXSTRICTRC rcStrict;
9105	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
9106	if (fSupportsNextRipSave)
9107	{
9108	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
9109	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
9110	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
9111	rcStrict = IEMExecDecodedInvlpga(pVCpu, cbInstr);
9112	}
9113	else
9114	{
9115	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
9116	rcStrict = IEMExecOne(pVCpu);
9117	}
9118
9119	if (rcStrict == VINF_IEM_RAISED_XCPT)
9120	{
9121	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9122	rcStrict = VINF_SUCCESS;
9123	}
9124	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
9125	return rcStrict;
9126	}
9127
9128
9129	/**
9130	* \#VMEXIT handler for STGI (SVM_EXIT_VMRUN). Conditional \#VMEXIT.
9131	*/
9132	HMSVM_EXIT_DECL hmR0SvmExitVmrun(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9133	{
9134	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9135	/* We shall import the entire state here, just in case we enter and continue execution of
9136	the nested-guest with hardware-assisted SVM in ring-0, we would be switching VMCBs and
9137	could lose lose part of CPU state. */
9138	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
9139
9140	VBOXSTRICTRC rcStrict;
9141	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
9142	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
9143	if (fSupportsNextRipSave)
9144	{
9145	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
9146	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
9147	rcStrict = IEMExecDecodedVmrun(pVCpu, cbInstr);
9148	}
9149	else
9150	{
9151	/* We use IEMExecOneBypassEx() here as it supresses attempt to continue emulating any
9152	instruction(s) when interrupt inhibition is set as part of emulating the VMRUN
9153	instruction itself, see @bugref{7243#c126} */
9154	rcStrict = IEMExecOneBypassEx(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), NULL /* pcbWritten */);
9155	}
9156	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
9157
9158	if (rcStrict == VINF_SUCCESS)
9159	{
9160	rcStrict = VINF_SVM_VMRUN;
9161	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_SVM_VMRUN_MASK);
9162	}
9163	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9164	{
9165	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9166	rcStrict = VINF_SUCCESS;
9167	}
9168	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
9169	return rcStrict;
9170	}
9171
9172
9173	/**
9174	* Nested-guest \#VMEXIT handler for debug exceptions (SVM_EXIT_XCPT_1).
9175	* Unconditional \#VMEXIT.
9176	*/
9177	HMSVM_EXIT_DECL hmR0SvmNestedExitXcptDB(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9178	{
9179	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9180	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
9181
9182	if (pVCpu->hm.s.Event.fPending)
9183	{
9184	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
9185	return VINF_EM_RAW_INJECT_TRPM_EVENT;
9186	}
9187
9188	hmR0SvmSetPendingXcptDB(pVCpu);
9189	return VINF_SUCCESS;
9190	}
9191
9192
9193	/**
9194	* Nested-guest \#VMEXIT handler for breakpoint exceptions (SVM_EXIT_XCPT_3).
9195	* Conditional \#VMEXIT.
9196	*/
9197	HMSVM_EXIT_DECL hmR0SvmNestedExitXcptBP(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9198	{
9199	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9200	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
9201
9202	SVMEVENT Event;
9203	Event.u = 0;
9204	Event.n.u1Valid = 1;
9205	Event.n.u3Type = SVM_EVENT_EXCEPTION;
9206	Event.n.u8Vector = X86_XCPT_BP;
9207	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
9208	return VINF_SUCCESS;
9209	}
9210	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
9211
9212	/** @} */
9213

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMSVMR0.cpp@ 95495

Download in other formats: