VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 73422

Last change on this file since 73422 was 73422, checked in by vboxsync, 7 years ago

VMM: Nested VMX: bugref:9180 bitfield macro nits.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 562.2 KB
Line 
1/* $Id: HMVMXR0.cpp 73422 2018-08-01 13:27:26Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/x86.h>
25#include <iprt/asm-amd64-x86.h>
26#include <iprt/thread.h>
27
28#include <VBox/vmm/pdmapi.h>
29#include <VBox/vmm/dbgf.h>
30#include <VBox/vmm/iem.h>
31#include <VBox/vmm/iom.h>
32#include <VBox/vmm/selm.h>
33#include <VBox/vmm/tm.h>
34#include <VBox/vmm/gim.h>
35#include <VBox/vmm/apic.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#include "HMInternal.h"
40#include <VBox/vmm/vm.h>
41#include "HMVMXR0.h"
42#include "dtrace/VBoxVMM.h"
43
44#ifdef DEBUG_ramshankar
45# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
47# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_CHECK_GUEST_STATE
49# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
50# define HMVMX_ALWAYS_TRAP_PF
51# define HMVMX_ALWAYS_FLUSH_TLB
52# define HMVMX_ALWAYS_SWAP_EFER
53#endif
54
55
56/*********************************************************************************************************************************
57* Defined Constants And Macros *
58*********************************************************************************************************************************/
59/** Use the function table. */
60#define HMVMX_USE_FUNCTION_TABLE
61
62/** Determine which tagged-TLB flush handler to use. */
63#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
64#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
65#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
66#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
67
68/** @name HMVMX_READ_XXX
69 * Flags to skip redundant reads of some common VMCS fields that are not part of
70 * the guest-CPU or VCPU state but are needed while handling VM-exits.
71 */
72#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
73#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
74#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
75#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
76#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
77#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
78#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
79/** @} */
80
81/**
82 * States of the VMCS.
83 *
84 * This does not reflect all possible VMCS states but currently only those
85 * needed for maintaining the VMCS consistently even when thread-context hooks
86 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
87 */
88#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
89#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
90#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
91
92/**
93 * Subset of the guest-CPU state that is kept by VMX R0 code while executing the
94 * guest using hardware-assisted VMX.
95 *
96 * This excludes state like GPRs (other than RSP) which are always are
97 * swapped and restored across the world-switch and also registers like EFER,
98 * MSR which cannot be modified by the guest without causing a VM-exit.
99 */
100#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
101 | CPUMCTX_EXTRN_RFLAGS \
102 | CPUMCTX_EXTRN_RSP \
103 | CPUMCTX_EXTRN_SREG_MASK \
104 | CPUMCTX_EXTRN_TABLE_MASK \
105 | CPUMCTX_EXTRN_KERNEL_GS_BASE \
106 | CPUMCTX_EXTRN_SYSCALL_MSRS \
107 | CPUMCTX_EXTRN_SYSENTER_MSRS \
108 | CPUMCTX_EXTRN_TSC_AUX \
109 | CPUMCTX_EXTRN_OTHER_MSRS \
110 | CPUMCTX_EXTRN_CR0 \
111 | CPUMCTX_EXTRN_CR3 \
112 | CPUMCTX_EXTRN_CR4 \
113 | CPUMCTX_EXTRN_DR7 \
114 | CPUMCTX_EXTRN_HM_VMX_MASK)
115
116/**
117 * Exception bitmap mask for real-mode guests (real-on-v86).
118 *
119 * We need to intercept all exceptions manually except:
120 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
121 * due to bugs in Intel CPUs.
122 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
123 * support.
124 */
125#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
126 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
127 | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \
128 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
129 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
130 | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
131 | RT_BIT(X86_XCPT_XF))
132
133/** Maximum VM-instruction error number. */
134#define HMVMX_INSTR_ERROR_MAX 28
135
136/** Profiling macro. */
137#ifdef HM_PROFILE_EXIT_DISPATCH
138# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
139# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
140#else
141# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
142# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
143#endif
144
145/** Assert that preemption is disabled or covered by thread-context hooks. */
146#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
147 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
148
149/** Assert that we haven't migrated CPUs when thread-context hooks are not
150 * used. */
151#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
152 || (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
153 ("Illegal migration! Entered on CPU %u Current %u\n", \
154 (a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
155
156/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
157 * context. */
158#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
159 ("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
160 (a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
161
162/** Helper macro for VM-exit handlers called unexpectedly. */
163#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
164 do { \
165 (a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
166 return VERR_VMX_UNEXPECTED_EXIT; \
167 } while (0)
168
169/** Macro for importing segment registers to the VMCS from the guest-CPU context. */
170#ifdef VMX_USE_CACHED_VMCS_ACCESSES
171# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
172 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
173 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
174#else
175# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
176 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
177 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
178#endif
179
180/** Macro for exporting segment registers to the VMCS from the guest-CPU context. */
181# define HMVMX_EXPORT_SREG(Sel, a_pCtxSelReg) \
182 hmR0VmxExportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
183 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
184
185
186/*********************************************************************************************************************************
187* Structures and Typedefs *
188*********************************************************************************************************************************/
189/**
190 * VMX transient state.
191 *
192 * A state structure for holding miscellaneous information across
193 * VMX non-root operation and restored after the transition.
194 */
195typedef struct VMXTRANSIENT
196{
197 /** The host's rflags/eflags. */
198 RTCCUINTREG fEFlags;
199#if HC_ARCH_BITS == 32
200 uint32_t u32Alignment0;
201#endif
202 /** The guest's TPR value used for TPR shadowing. */
203 uint8_t u8GuestTpr;
204 /** Alignment. */
205 uint8_t abAlignment0[7];
206
207 /** The basic VM-exit reason. */
208 uint16_t uExitReason;
209 /** Alignment. */
210 uint16_t u16Alignment0;
211 /** The VM-exit interruption error code. */
212 uint32_t uExitIntErrorCode;
213 /** The VM-exit exit code qualification. */
214 uint64_t uExitQualification;
215
216 /** The VM-exit interruption-information field. */
217 uint32_t uExitIntInfo;
218 /** The VM-exit instruction-length field. */
219 uint32_t cbInstr;
220 /** The VM-exit instruction-information field. */
221 union
222 {
223 /** Plain unsigned int representation. */
224 uint32_t u;
225 /** INS and OUTS information. */
226 struct
227 {
228 uint32_t u7Reserved0 : 7;
229 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
230 uint32_t u3AddrSize : 3;
231 uint32_t u5Reserved1 : 5;
232 /** The segment register (X86_SREG_XXX). */
233 uint32_t iSegReg : 3;
234 uint32_t uReserved2 : 14;
235 } StrIo;
236 /** INVEPT, INVVPID, INVPCID information. */
237 struct
238 {
239 /** Scaling; 0=no scaling, 1=scale-by-2, 2=scale-by-4, 3=scale-by-8. */
240 uint32_t u2Scaling : 2;
241 uint32_t u5Reserved0 : 5;
242 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
243 uint32_t u3AddrSize : 3;
244 uint32_t u1Reserved0 : 1;
245 uint32_t u4Reserved0 : 4;
246 /** The segment register (X86_SREG_XXX). */
247 uint32_t iSegReg : 3;
248 /** The index register (X86_GREG_XXX). */
249 uint32_t iIdxReg : 4;
250 /** Set if index register is invalid. */
251 uint32_t fIdxRegValid : 1;
252 /** The base register (X86_GREG_XXX). */
253 uint32_t iBaseReg : 4;
254 /** Set if base register is invalid. */
255 uint32_t fBaseRegValid : 1;
256 /** Register 2 (X86_GREG_XXX). */
257 uint32_t iReg2 : 4;
258 } Inv;
259 } ExitInstrInfo;
260 /** Whether the VM-entry failed or not. */
261 bool fVMEntryFailed;
262 /** Alignment. */
263 uint8_t abAlignment1[3];
264
265 /** The VM-entry interruption-information field. */
266 uint32_t uEntryIntInfo;
267 /** The VM-entry exception error code field. */
268 uint32_t uEntryXcptErrorCode;
269 /** The VM-entry instruction length field. */
270 uint32_t cbEntryInstr;
271
272 /** IDT-vectoring information field. */
273 uint32_t uIdtVectoringInfo;
274 /** IDT-vectoring error code. */
275 uint32_t uIdtVectoringErrorCode;
276
277 /** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
278 uint32_t fVmcsFieldsRead;
279
280 /** Whether the guest debug state was active at the time of VM-exit. */
281 bool fWasGuestDebugStateActive;
282 /** Whether the hyper debug state was active at the time of VM-exit. */
283 bool fWasHyperDebugStateActive;
284 /** Whether TSC-offsetting should be setup before VM-entry. */
285 bool fUpdateTscOffsettingAndPreemptTimer;
286 /** Whether the VM-exit was caused by a page-fault during delivery of a
287 * contributory exception or a page-fault. */
288 bool fVectoringDoublePF;
289 /** Whether the VM-exit was caused by a page-fault during delivery of an
290 * external interrupt or NMI. */
291 bool fVectoringPF;
292} VMXTRANSIENT;
293AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
294AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
295AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
296AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
297AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
298/** Pointer to VMX transient state. */
299typedef VMXTRANSIENT *PVMXTRANSIENT;
300
301
302/**
303 * MSR-bitmap read permissions.
304 */
305typedef enum VMXMSREXITREAD
306{
307 /** Reading this MSR causes a VM-exit. */
308 VMXMSREXIT_INTERCEPT_READ = 0xb,
309 /** Reading this MSR does not cause a VM-exit. */
310 VMXMSREXIT_PASSTHRU_READ
311} VMXMSREXITREAD;
312/** Pointer to MSR-bitmap read permissions. */
313typedef VMXMSREXITREAD* PVMXMSREXITREAD;
314
315/**
316 * MSR-bitmap write permissions.
317 */
318typedef enum VMXMSREXITWRITE
319{
320 /** Writing to this MSR causes a VM-exit. */
321 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
322 /** Writing to this MSR does not cause a VM-exit. */
323 VMXMSREXIT_PASSTHRU_WRITE
324} VMXMSREXITWRITE;
325/** Pointer to MSR-bitmap write permissions. */
326typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
327
328
329/**
330 * VMX VM-exit handler.
331 *
332 * @returns Strict VBox status code (i.e. informational status codes too).
333 * @param pVCpu The cross context virtual CPU structure.
334 * @param pVmxTransient Pointer to the VMX-transient structure.
335 */
336#ifndef HMVMX_USE_FUNCTION_TABLE
337typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
338#else
339typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
340/** Pointer to VM-exit handler. */
341typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
342#endif
343
344/**
345 * VMX VM-exit handler, non-strict status code.
346 *
347 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
348 *
349 * @returns VBox status code, no informational status code returned.
350 * @param pVCpu The cross context virtual CPU structure.
351 * @param pVmxTransient Pointer to the VMX-transient structure.
352 *
353 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
354 * use of that status code will be replaced with VINF_EM_SOMETHING
355 * later when switching over to IEM.
356 */
357#ifndef HMVMX_USE_FUNCTION_TABLE
358typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
359#else
360typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
361#endif
362
363
364/*********************************************************************************************************************************
365* Internal Functions *
366*********************************************************************************************************************************/
367static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush);
368static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr);
369static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
370static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat);
371static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
372 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState);
373#if HC_ARCH_BITS == 32
374static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
375#endif
376#ifndef HMVMX_USE_FUNCTION_TABLE
377DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
378# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
379# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
380#else
381# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
382# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
383#endif
384
385
386/** @name VM-exit handlers.
387 * @{
388 */
389static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
390static FNVMXEXITHANDLER hmR0VmxExitExtInt;
391static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
392static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
393static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
394static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
395static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
396static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
397static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
398static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
399static FNVMXEXITHANDLER hmR0VmxExitCpuid;
400static FNVMXEXITHANDLER hmR0VmxExitGetsec;
401static FNVMXEXITHANDLER hmR0VmxExitHlt;
402static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
403static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
404static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
405static FNVMXEXITHANDLER hmR0VmxExitVmcall;
406static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
407static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
408static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
409static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
410static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
411static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
412static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
413static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
414static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
415static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
416static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
417static FNVMXEXITHANDLER hmR0VmxExitMwait;
418static FNVMXEXITHANDLER hmR0VmxExitMtf;
419static FNVMXEXITHANDLER hmR0VmxExitMonitor;
420static FNVMXEXITHANDLER hmR0VmxExitPause;
421static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
422static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
423static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
424static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
425static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
426static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
427static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
428static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
429static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
430static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
431static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
432static FNVMXEXITHANDLER hmR0VmxExitRdrand;
433static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
434/** @} */
435
436static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
437static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
438static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
439static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
440static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
441static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
442static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
443static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu);
444
445
446/*********************************************************************************************************************************
447* Global Variables *
448*********************************************************************************************************************************/
449#ifdef HMVMX_USE_FUNCTION_TABLE
450
451/**
452 * VMX_EXIT dispatch table.
453 */
454static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
455{
456 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
457 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
458 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
459 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
460 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
461 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
462 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
463 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
464 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
465 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
466 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
467 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
468 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
469 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
470 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
471 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
472 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
473 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
474 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
475 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
476 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
477 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
478 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
479 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
480 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
481 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
482 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
483 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
484 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
485 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
486 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
487 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
488 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
489 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
490 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
491 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
492 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
493 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
494 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
495 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
496 /* 40 UNDEFINED */ hmR0VmxExitPause,
497 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
498 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
499 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
500 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
501 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
502 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
503 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
504 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
505 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
506 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
507 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
508 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
509 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
510 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
511 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
512 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
513 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
514 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
515 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
516 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
517 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
518 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
519 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
520 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
521};
522#endif /* HMVMX_USE_FUNCTION_TABLE */
523
524#ifdef VBOX_STRICT
525static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
526{
527 /* 0 */ "(Not Used)",
528 /* 1 */ "VMCALL executed in VMX root operation.",
529 /* 2 */ "VMCLEAR with invalid physical address.",
530 /* 3 */ "VMCLEAR with VMXON pointer.",
531 /* 4 */ "VMLAUNCH with non-clear VMCS.",
532 /* 5 */ "VMRESUME with non-launched VMCS.",
533 /* 6 */ "VMRESUME after VMXOFF",
534 /* 7 */ "VM-entry with invalid control fields.",
535 /* 8 */ "VM-entry with invalid host state fields.",
536 /* 9 */ "VMPTRLD with invalid physical address.",
537 /* 10 */ "VMPTRLD with VMXON pointer.",
538 /* 11 */ "VMPTRLD with incorrect revision identifier.",
539 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
540 /* 13 */ "VMWRITE to read-only VMCS component.",
541 /* 14 */ "(Not Used)",
542 /* 15 */ "VMXON executed in VMX root operation.",
543 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
544 /* 17 */ "VM-entry with non-launched executing VMCS.",
545 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
546 /* 19 */ "VMCALL with non-clear VMCS.",
547 /* 20 */ "VMCALL with invalid VM-exit control fields.",
548 /* 21 */ "(Not Used)",
549 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
550 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
551 /* 24 */ "VMCALL with invalid SMM-monitor features.",
552 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
553 /* 26 */ "VM-entry with events blocked by MOV SS.",
554 /* 27 */ "(Not Used)",
555 /* 28 */ "Invalid operand to INVEPT/INVVPID."
556};
557#endif /* VBOX_STRICT */
558
559
560
561/**
562 * Updates the VM's last error record.
563 *
564 * If there was a VMX instruction error, reads the error data from the VMCS and
565 * updates VCPU's last error record as well.
566 *
567 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
568 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
569 * VERR_VMX_INVALID_VMCS_FIELD.
570 * @param rc The error code.
571 */
572static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
573{
574 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
575 || rc == VERR_VMX_UNABLE_TO_START_VM)
576 {
577 AssertPtrReturnVoid(pVCpu);
578 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
579 }
580 pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
581}
582
583
584/**
585 * Reads the VM-entry interruption-information field from the VMCS into the VMX
586 * transient structure.
587 *
588 * @returns VBox status code.
589 * @param pVmxTransient Pointer to the VMX transient structure.
590 *
591 * @remarks No-long-jump zone!!!
592 */
593DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
594{
595 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
596 AssertRCReturn(rc, rc);
597 return VINF_SUCCESS;
598}
599
600#ifdef VBOX_STRICT
601/**
602 * Reads the VM-entry exception error code field from the VMCS into
603 * the VMX transient structure.
604 *
605 * @returns VBox status code.
606 * @param pVmxTransient Pointer to the VMX transient structure.
607 *
608 * @remarks No-long-jump zone!!!
609 */
610DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
611{
612 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
613 AssertRCReturn(rc, rc);
614 return VINF_SUCCESS;
615}
616
617
618/**
619 * Reads the VM-entry exception error code field from the VMCS into
620 * the VMX transient structure.
621 *
622 * @returns VBox status code.
623 * @param pVmxTransient Pointer to the VMX transient structure.
624 *
625 * @remarks No-long-jump zone!!!
626 */
627DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
628{
629 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
630 AssertRCReturn(rc, rc);
631 return VINF_SUCCESS;
632}
633#endif /* VBOX_STRICT */
634
635
636/**
637 * Reads the VM-exit interruption-information field from the VMCS into the VMX
638 * transient structure.
639 *
640 * @returns VBox status code.
641 * @param pVmxTransient Pointer to the VMX transient structure.
642 */
643DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
644{
645 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
646 {
647 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
648 AssertRCReturn(rc,rc);
649 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_INFO;
650 }
651 return VINF_SUCCESS;
652}
653
654
655/**
656 * Reads the VM-exit interruption error code from the VMCS into the VMX
657 * transient structure.
658 *
659 * @returns VBox status code.
660 * @param pVmxTransient Pointer to the VMX transient structure.
661 */
662DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
663{
664 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
665 {
666 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
667 AssertRCReturn(rc, rc);
668 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
669 }
670 return VINF_SUCCESS;
671}
672
673
674/**
675 * Reads the VM-exit instruction length field from the VMCS into the VMX
676 * transient structure.
677 *
678 * @returns VBox status code.
679 * @param pVmxTransient Pointer to the VMX transient structure.
680 */
681DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
682{
683 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
684 {
685 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
686 AssertRCReturn(rc, rc);
687 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_LEN;
688 }
689 return VINF_SUCCESS;
690}
691
692
693/**
694 * Reads the VM-exit instruction-information field from the VMCS into
695 * the VMX transient structure.
696 *
697 * @returns VBox status code.
698 * @param pVmxTransient Pointer to the VMX transient structure.
699 */
700DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
701{
702 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
703 {
704 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
705 AssertRCReturn(rc, rc);
706 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_INFO;
707 }
708 return VINF_SUCCESS;
709}
710
711
712/**
713 * Reads the exit code qualification from the VMCS into the VMX transient
714 * structure.
715 *
716 * @returns VBox status code.
717 * @param pVCpu The cross context virtual CPU structure of the
718 * calling EMT. (Required for the VMCS cache case.)
719 * @param pVmxTransient Pointer to the VMX transient structure.
720 */
721DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
722{
723 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
724 {
725 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
726 AssertRCReturn(rc, rc);
727 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION;
728 }
729 return VINF_SUCCESS;
730}
731
732
733/**
734 * Reads the IDT-vectoring information field from the VMCS into the VMX
735 * transient structure.
736 *
737 * @returns VBox status code.
738 * @param pVmxTransient Pointer to the VMX transient structure.
739 *
740 * @remarks No-long-jump zone!!!
741 */
742DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
743{
744 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
745 {
746 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
747 AssertRCReturn(rc, rc);
748 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_INFO;
749 }
750 return VINF_SUCCESS;
751}
752
753
754/**
755 * Reads the IDT-vectoring error code from the VMCS into the VMX
756 * transient structure.
757 *
758 * @returns VBox status code.
759 * @param pVmxTransient Pointer to the VMX transient structure.
760 */
761DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
762{
763 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
764 {
765 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
766 AssertRCReturn(rc, rc);
767 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
768 }
769 return VINF_SUCCESS;
770}
771
772
773/**
774 * Enters VMX root mode operation on the current CPU.
775 *
776 * @returns VBox status code.
777 * @param pVM The cross context VM structure. Can be
778 * NULL, after a resume.
779 * @param HCPhysCpuPage Physical address of the VMXON region.
780 * @param pvCpuPage Pointer to the VMXON region.
781 */
782static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
783{
784 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
785 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
786 Assert(pvCpuPage);
787 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
788
789 if (pVM)
790 {
791 /* Write the VMCS revision dword to the VMXON region. */
792 *(uint32_t *)pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
793 }
794
795 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
796 RTCCUINTREG fEFlags = ASMIntDisableFlags();
797
798 /* Enable the VMX bit in CR4 if necessary. */
799 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
800
801 /* Enter VMX root mode. */
802 int rc = VMXEnable(HCPhysCpuPage);
803 if (RT_FAILURE(rc))
804 {
805 if (!(uOldCr4 & X86_CR4_VMXE))
806 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
807
808 if (pVM)
809 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
810 }
811
812 /* Restore interrupts. */
813 ASMSetFlags(fEFlags);
814 return rc;
815}
816
817
818/**
819 * Exits VMX root mode operation on the current CPU.
820 *
821 * @returns VBox status code.
822 */
823static int hmR0VmxLeaveRootMode(void)
824{
825 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
826
827 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
828 RTCCUINTREG fEFlags = ASMIntDisableFlags();
829
830 /* If we're for some reason not in VMX root mode, then don't leave it. */
831 RTCCUINTREG uHostCR4 = ASMGetCR4();
832
833 int rc;
834 if (uHostCR4 & X86_CR4_VMXE)
835 {
836 /* Exit VMX root mode and clear the VMX bit in CR4. */
837 VMXDisable();
838 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
839 rc = VINF_SUCCESS;
840 }
841 else
842 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
843
844 /* Restore interrupts. */
845 ASMSetFlags(fEFlags);
846 return rc;
847}
848
849
850/**
851 * Allocates and maps one physically contiguous page. The allocated page is
852 * zero'd out. (Used by various VT-x structures).
853 *
854 * @returns IPRT status code.
855 * @param pMemObj Pointer to the ring-0 memory object.
856 * @param ppVirt Where to store the virtual address of the
857 * allocation.
858 * @param pHCPhys Where to store the physical address of the
859 * allocation.
860 */
861static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
862{
863 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
864 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
865 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
866
867 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
868 if (RT_FAILURE(rc))
869 return rc;
870 *ppVirt = RTR0MemObjAddress(*pMemObj);
871 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
872 ASMMemZero32(*ppVirt, PAGE_SIZE);
873 return VINF_SUCCESS;
874}
875
876
877/**
878 * Frees and unmaps an allocated physical page.
879 *
880 * @param pMemObj Pointer to the ring-0 memory object.
881 * @param ppVirt Where to re-initialize the virtual address of
882 * allocation as 0.
883 * @param pHCPhys Where to re-initialize the physical address of the
884 * allocation as 0.
885 */
886static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
887{
888 AssertPtr(pMemObj);
889 AssertPtr(ppVirt);
890 AssertPtr(pHCPhys);
891 if (*pMemObj != NIL_RTR0MEMOBJ)
892 {
893 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
894 AssertRC(rc);
895 *pMemObj = NIL_RTR0MEMOBJ;
896 *ppVirt = 0;
897 *pHCPhys = 0;
898 }
899}
900
901
902/**
903 * Worker function to free VT-x related structures.
904 *
905 * @returns IPRT status code.
906 * @param pVM The cross context VM structure.
907 */
908static void hmR0VmxStructsFree(PVM pVM)
909{
910 for (VMCPUID i = 0; i < pVM->cCpus; i++)
911 {
912 PVMCPU pVCpu = &pVM->aCpus[i];
913 AssertPtr(pVCpu);
914
915 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
916 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
917
918 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
919 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
920
921 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
922 }
923
924 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
925#ifdef VBOX_WITH_CRASHDUMP_MAGIC
926 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
927#endif
928}
929
930
931/**
932 * Worker function to allocate VT-x related VM structures.
933 *
934 * @returns IPRT status code.
935 * @param pVM The cross context VM structure.
936 */
937static int hmR0VmxStructsAlloc(PVM pVM)
938{
939 /*
940 * Initialize members up-front so we can cleanup properly on allocation failure.
941 */
942#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
943 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
944 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
945 pVM->hm.s.vmx.HCPhys##a_Name = 0;
946
947#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
948 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
949 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
950 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
951
952#ifdef VBOX_WITH_CRASHDUMP_MAGIC
953 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
954#endif
955 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
956
957 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
958 for (VMCPUID i = 0; i < pVM->cCpus; i++)
959 {
960 PVMCPU pVCpu = &pVM->aCpus[i];
961 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
962 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
963 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
964 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
965 }
966#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
967#undef VMXLOCAL_INIT_VM_MEMOBJ
968
969 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
970 AssertReturnStmt(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE) <= PAGE_SIZE,
971 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
972 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
973
974 /*
975 * Allocate all the VT-x structures.
976 */
977 int rc = VINF_SUCCESS;
978#ifdef VBOX_WITH_CRASHDUMP_MAGIC
979 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
980 if (RT_FAILURE(rc))
981 goto cleanup;
982 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
983 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
984#endif
985
986 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
987 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
988 {
989 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
990 &pVM->hm.s.vmx.HCPhysApicAccess);
991 if (RT_FAILURE(rc))
992 goto cleanup;
993 }
994
995 /*
996 * Initialize per-VCPU VT-x structures.
997 */
998 for (VMCPUID i = 0; i < pVM->cCpus; i++)
999 {
1000 PVMCPU pVCpu = &pVM->aCpus[i];
1001 AssertPtr(pVCpu);
1002
1003 /* Allocate the VM control structure (VMCS). */
1004 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1005 if (RT_FAILURE(rc))
1006 goto cleanup;
1007
1008 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1009 if ( PDMHasApic(pVM)
1010 && (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1011 {
1012 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1013 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1014 if (RT_FAILURE(rc))
1015 goto cleanup;
1016 }
1017
1018 /*
1019 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1020 * transparent accesses of specific MSRs.
1021 *
1022 * If the condition for enabling MSR bitmaps changes here, don't forget to
1023 * update HMAreMsrBitmapsAvailable().
1024 */
1025 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1026 {
1027 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1028 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1029 if (RT_FAILURE(rc))
1030 goto cleanup;
1031 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1032 }
1033
1034 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1035 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1036 if (RT_FAILURE(rc))
1037 goto cleanup;
1038
1039 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1040 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1041 if (RT_FAILURE(rc))
1042 goto cleanup;
1043 }
1044
1045 return VINF_SUCCESS;
1046
1047cleanup:
1048 hmR0VmxStructsFree(pVM);
1049 return rc;
1050}
1051
1052
1053/**
1054 * Does global VT-x initialization (called during module initialization).
1055 *
1056 * @returns VBox status code.
1057 */
1058VMMR0DECL(int) VMXR0GlobalInit(void)
1059{
1060#ifdef HMVMX_USE_FUNCTION_TABLE
1061 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1062# ifdef VBOX_STRICT
1063 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1064 Assert(g_apfnVMExitHandlers[i]);
1065# endif
1066#endif
1067 return VINF_SUCCESS;
1068}
1069
1070
1071/**
1072 * Does global VT-x termination (called during module termination).
1073 */
1074VMMR0DECL(void) VMXR0GlobalTerm()
1075{
1076 /* Nothing to do currently. */
1077}
1078
1079
1080/**
1081 * Sets up and activates VT-x on the current CPU.
1082 *
1083 * @returns VBox status code.
1084 * @param pHostCpu Pointer to the global CPU info struct.
1085 * @param pVM The cross context VM structure. Can be
1086 * NULL after a host resume operation.
1087 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1088 * fEnabledByHost is @c true).
1089 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1090 * @a fEnabledByHost is @c true).
1091 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1092 * enable VT-x on the host.
1093 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1094 */
1095VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1096 void *pvMsrs)
1097{
1098 Assert(pHostCpu);
1099 Assert(pvMsrs);
1100 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1101
1102 /* Enable VT-x if it's not already enabled by the host. */
1103 if (!fEnabledByHost)
1104 {
1105 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1106 if (RT_FAILURE(rc))
1107 return rc;
1108 }
1109
1110 /*
1111 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
1112 * using EPTPs) so we don't retain any stale guest-physical mappings which won't get
1113 * invalidated when flushing by VPID.
1114 */
1115 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1116 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1117 {
1118 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
1119 pHostCpu->fFlushAsidBeforeUse = false;
1120 }
1121 else
1122 pHostCpu->fFlushAsidBeforeUse = true;
1123
1124 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1125 ++pHostCpu->cTlbFlushes;
1126
1127 return VINF_SUCCESS;
1128}
1129
1130
1131/**
1132 * Deactivates VT-x on the current CPU.
1133 *
1134 * @returns VBox status code.
1135 * @param pHostCpu Pointer to the global CPU info struct.
1136 * @param pvCpuPage Pointer to the VMXON region.
1137 * @param HCPhysCpuPage Physical address of the VMXON region.
1138 *
1139 * @remarks This function should never be called when SUPR0EnableVTx() or
1140 * similar was used to enable VT-x on the host.
1141 */
1142VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1143{
1144 RT_NOREF3(pHostCpu, pvCpuPage, HCPhysCpuPage);
1145
1146 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1147 return hmR0VmxLeaveRootMode();
1148}
1149
1150
1151/**
1152 * Sets the permission bits for the specified MSR in the MSR bitmap.
1153 *
1154 * @param pVCpu The cross context virtual CPU structure.
1155 * @param uMsr The MSR value.
1156 * @param enmRead Whether reading this MSR causes a VM-exit.
1157 * @param enmWrite Whether writing this MSR causes a VM-exit.
1158 */
1159static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1160{
1161 int32_t iBit;
1162 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1163
1164 /*
1165 * Layout:
1166 * 0x000 - 0x3ff - Low MSR read bits
1167 * 0x400 - 0x7ff - High MSR read bits
1168 * 0x800 - 0xbff - Low MSR write bits
1169 * 0xc00 - 0xfff - High MSR write bits
1170 */
1171 if (uMsr <= 0x00001fff)
1172 iBit = uMsr;
1173 else if (uMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1174 {
1175 iBit = uMsr - UINT32_C(0xc0000000);
1176 pbMsrBitmap += 0x400;
1177 }
1178 else
1179 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1180
1181 Assert(iBit <= 0x1fff);
1182 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1183 ASMBitSet(pbMsrBitmap, iBit);
1184 else
1185 ASMBitClear(pbMsrBitmap, iBit);
1186
1187 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1188 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1189 else
1190 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1191}
1192
1193
1194#ifdef VBOX_STRICT
1195/**
1196 * Gets the permission bits for the specified MSR in the MSR bitmap.
1197 *
1198 * @returns VBox status code.
1199 * @retval VINF_SUCCESS if the specified MSR is found.
1200 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1201 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1202 *
1203 * @param pVCpu The cross context virtual CPU structure.
1204 * @param uMsr The MSR.
1205 * @param penmRead Where to store the read permissions.
1206 * @param penmWrite Where to store the write permissions.
1207 */
1208static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1209{
1210 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1211 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1212 int32_t iBit;
1213 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1214
1215 /* See hmR0VmxSetMsrPermission() for the layout. */
1216 if (uMsr <= 0x00001fff)
1217 iBit = uMsr;
1218 else if ( uMsr >= 0xc0000000
1219 && uMsr <= 0xc0001fff)
1220 {
1221 iBit = (uMsr - 0xc0000000);
1222 pbMsrBitmap += 0x400;
1223 }
1224 else
1225 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1226
1227 Assert(iBit <= 0x1fff);
1228 if (ASMBitTest(pbMsrBitmap, iBit))
1229 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1230 else
1231 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1232
1233 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1234 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1235 else
1236 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1237 return VINF_SUCCESS;
1238}
1239#endif /* VBOX_STRICT */
1240
1241
1242/**
1243 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1244 * area.
1245 *
1246 * @returns VBox status code.
1247 * @param pVCpu The cross context virtual CPU structure.
1248 * @param cMsrs The number of MSRs.
1249 */
1250static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1251{
1252 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1253 uint64_t const uVmxMiscMsr = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc;
1254 uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(uVmxMiscMsr);
1255 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1256 {
1257 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1258 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1259 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1260 }
1261
1262 /* Update number of guest MSRs to load/store across the world-switch. */
1263 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1264 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1265
1266 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1267 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1268 AssertRCReturn(rc, rc);
1269
1270 /* Update the VCPU's copy of the MSR count. */
1271 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1272
1273 return VINF_SUCCESS;
1274}
1275
1276
1277/**
1278 * Adds a new (or updates the value of an existing) guest/host MSR
1279 * pair to be swapped during the world-switch as part of the
1280 * auto-load/store MSR area in the VMCS.
1281 *
1282 * @returns VBox status code.
1283 * @param pVCpu The cross context virtual CPU structure.
1284 * @param uMsr The MSR.
1285 * @param uGuestMsrValue Value of the guest MSR.
1286 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1287 * necessary.
1288 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1289 * its value was updated. Optional, can be NULL.
1290 */
1291static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1292 bool *pfAddedAndUpdated)
1293{
1294 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1295 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1296 uint32_t i;
1297 for (i = 0; i < cMsrs; i++)
1298 {
1299 if (pGuestMsr->u32Msr == uMsr)
1300 break;
1301 pGuestMsr++;
1302 }
1303
1304 bool fAdded = false;
1305 if (i == cMsrs)
1306 {
1307 ++cMsrs;
1308 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1309 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1310
1311 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1312 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1313 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1314
1315 fAdded = true;
1316 }
1317
1318 /* Update the MSR values in the auto-load/store MSR area. */
1319 pGuestMsr->u32Msr = uMsr;
1320 pGuestMsr->u64Value = uGuestMsrValue;
1321
1322 /* Create/update the MSR slot in the host MSR area. */
1323 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1324 pHostMsr += i;
1325 pHostMsr->u32Msr = uMsr;
1326
1327 /*
1328 * Update the host MSR only when requested by the caller AND when we're
1329 * adding it to the auto-load/store area. Otherwise, it would have been
1330 * updated by hmR0VmxExportHostMsrs(). We do this for performance reasons.
1331 */
1332 bool fUpdatedMsrValue = false;
1333 if ( fAdded
1334 && fUpdateHostMsr)
1335 {
1336 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1337 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1338 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1339 fUpdatedMsrValue = true;
1340 }
1341
1342 if (pfAddedAndUpdated)
1343 *pfAddedAndUpdated = fUpdatedMsrValue;
1344 return VINF_SUCCESS;
1345}
1346
1347
1348/**
1349 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1350 * auto-load/store MSR area in the VMCS.
1351 *
1352 * @returns VBox status code.
1353 * @param pVCpu The cross context virtual CPU structure.
1354 * @param uMsr The MSR.
1355 */
1356static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1357{
1358 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1359 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1360 for (uint32_t i = 0; i < cMsrs; i++)
1361 {
1362 /* Find the MSR. */
1363 if (pGuestMsr->u32Msr == uMsr)
1364 {
1365 /* If it's the last MSR, simply reduce the count. */
1366 if (i == cMsrs - 1)
1367 {
1368 --cMsrs;
1369 break;
1370 }
1371
1372 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1373 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1374 pLastGuestMsr += cMsrs - 1;
1375 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1376 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1377
1378 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1379 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1380 pLastHostMsr += cMsrs - 1;
1381 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1382 pHostMsr->u64Value = pLastHostMsr->u64Value;
1383 --cMsrs;
1384 break;
1385 }
1386 pGuestMsr++;
1387 }
1388
1389 /* Update the VMCS if the count changed (meaning the MSR was found). */
1390 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1391 {
1392 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1393 AssertRCReturn(rc, rc);
1394
1395 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1396 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1397 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1398
1399 Log4Func(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1400 return VINF_SUCCESS;
1401 }
1402
1403 return VERR_NOT_FOUND;
1404}
1405
1406
1407/**
1408 * Checks if the specified guest MSR is part of the auto-load/store area in
1409 * the VMCS.
1410 *
1411 * @returns true if found, false otherwise.
1412 * @param pVCpu The cross context virtual CPU structure.
1413 * @param uMsr The MSR to find.
1414 */
1415static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1416{
1417 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1418 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1419
1420 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1421 {
1422 if (pGuestMsr->u32Msr == uMsr)
1423 return true;
1424 }
1425 return false;
1426}
1427
1428
1429/**
1430 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1431 *
1432 * @param pVCpu The cross context virtual CPU structure.
1433 *
1434 * @remarks No-long-jump zone!!!
1435 */
1436static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1437{
1438 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1439 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1440 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1441 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1442
1443 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1444 {
1445 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1446
1447 /*
1448 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1449 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1450 */
1451 if (pHostMsr->u32Msr == MSR_K6_EFER)
1452 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1453 else
1454 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1455 }
1456
1457 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1458}
1459
1460
1461/**
1462 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1463 * perform lazy restoration of the host MSRs while leaving VT-x.
1464 *
1465 * @param pVCpu The cross context virtual CPU structure.
1466 *
1467 * @remarks No-long-jump zone!!!
1468 */
1469static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1470{
1471 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1472
1473 /*
1474 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1475 */
1476 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1477 {
1478 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1479#if HC_ARCH_BITS == 64
1480 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1481 {
1482 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1483 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1484 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1485 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1486 }
1487#endif
1488 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1489 }
1490}
1491
1492
1493/**
1494 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1495 * lazily while leaving VT-x.
1496 *
1497 * @returns true if it does, false otherwise.
1498 * @param pVCpu The cross context virtual CPU structure.
1499 * @param uMsr The MSR to check.
1500 */
1501static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1502{
1503 NOREF(pVCpu);
1504#if HC_ARCH_BITS == 64
1505 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1506 {
1507 switch (uMsr)
1508 {
1509 case MSR_K8_LSTAR:
1510 case MSR_K6_STAR:
1511 case MSR_K8_SF_MASK:
1512 case MSR_K8_KERNEL_GS_BASE:
1513 return true;
1514 }
1515 }
1516#else
1517 RT_NOREF(pVCpu, uMsr);
1518#endif
1519 return false;
1520}
1521
1522
1523/**
1524 * Loads a set of guests MSRs to allow read/passthru to the guest.
1525 *
1526 * The name of this function is slightly confusing. This function does NOT
1527 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1528 * common prefix for functions dealing with "lazy restoration" of the shared
1529 * MSRs.
1530 *
1531 * @param pVCpu The cross context virtual CPU structure.
1532 *
1533 * @remarks No-long-jump zone!!!
1534 */
1535static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1536{
1537 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1538 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1539
1540 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1541#if HC_ARCH_BITS == 64
1542 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1543 {
1544 /*
1545 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1546 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1547 * we can skip a few MSR writes.
1548 *
1549 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1550 * guest MSR values in the guest-CPU context might be different to what's currently
1551 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1552 * CPU, see @bugref{8728}.
1553 */
1554 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1555 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1556 && pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1557 && pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1558 && pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1559 && pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1560 {
1561#ifdef VBOX_STRICT
1562 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1563 Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
1564 Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
1565 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
1566#endif
1567 }
1568 else
1569 {
1570 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
1571 ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
1572 ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
1573 ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
1574 }
1575 }
1576#endif
1577 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1578}
1579
1580
1581/**
1582 * Performs lazy restoration of the set of host MSRs if they were previously
1583 * loaded with guest MSR values.
1584 *
1585 * @param pVCpu The cross context virtual CPU structure.
1586 *
1587 * @remarks No-long-jump zone!!!
1588 * @remarks The guest MSRs should have been saved back into the guest-CPU
1589 * context by hmR0VmxImportGuestState()!!!
1590 */
1591static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1592{
1593 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1594 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1595
1596 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1597 {
1598 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1599#if HC_ARCH_BITS == 64
1600 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1601 {
1602 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1603 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1604 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1605 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1606 }
1607#endif
1608 }
1609 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1610}
1611
1612
1613/**
1614 * Verifies that our cached values of the VMCS fields are all consistent with
1615 * what's actually present in the VMCS.
1616 *
1617 * @returns VBox status code.
1618 * @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
1619 * @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
1620 * VMCS content. HMCPU error-field is
1621 * updated, see VMX_VCI_XXX.
1622 * @param pVCpu The cross context virtual CPU structure.
1623 */
1624static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1625{
1626 uint32_t u32Val;
1627 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1628 AssertRCReturn(rc, rc);
1629 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32EntryCtls == u32Val,
1630 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1631 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
1632 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1633
1634 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1635 AssertRCReturn(rc, rc);
1636 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ExitCtls == u32Val,
1637 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1638 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
1639 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1640
1641 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1642 AssertRCReturn(rc, rc);
1643 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32PinCtls == u32Val,
1644 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1645 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
1646 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1647
1648 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1649 AssertRCReturn(rc, rc);
1650 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls == u32Val,
1651 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1652 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
1653 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1654
1655 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
1656 {
1657 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1658 AssertRCReturn(rc, rc);
1659 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1660 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1661 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
1662 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1663 }
1664
1665 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
1666 AssertRCReturn(rc, rc);
1667 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32XcptBitmap == u32Val,
1668 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap, u32Val),
1669 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
1670 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1671
1672 uint64_t u64Val;
1673 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
1674 AssertRCReturn(rc, rc);
1675 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u64TscOffset == u64Val,
1676 ("Cache=%#RX64 VMCS=%#RX64\n", pVCpu->hm.s.vmx.u64TscOffset, u64Val),
1677 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
1678 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1679
1680 return VINF_SUCCESS;
1681}
1682
1683
1684#ifdef VBOX_STRICT
1685/**
1686 * Verifies that our cached host EFER value has not changed
1687 * since we cached it.
1688 *
1689 * @param pVCpu The cross context virtual CPU structure.
1690 */
1691static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1692{
1693 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1694
1695 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
1696 {
1697 uint64_t u64Val;
1698 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1699 AssertRC(rc);
1700
1701 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1702 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1703 }
1704}
1705
1706
1707/**
1708 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1709 * VMCS are correct.
1710 *
1711 * @param pVCpu The cross context virtual CPU structure.
1712 */
1713static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1714{
1715 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1716
1717 /* Verify MSR counts in the VMCS are what we think it should be. */
1718 uint32_t cMsrs;
1719 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1720 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1721
1722 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1723 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1724
1725 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1726 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1727
1728 PCVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1729 PCVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1730 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1731 {
1732 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1733 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1734 pGuestMsr->u32Msr, cMsrs));
1735
1736 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1737 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1738 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1739
1740 /* Verify that the permissions are as expected in the MSR bitmap. */
1741 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1742 {
1743 VMXMSREXITREAD enmRead;
1744 VMXMSREXITWRITE enmWrite;
1745 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1746 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1747 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1748 {
1749 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1750 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1751 }
1752 else
1753 {
1754 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1755 pGuestMsr->u32Msr, cMsrs));
1756 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1757 pGuestMsr->u32Msr, cMsrs));
1758 }
1759 }
1760 }
1761}
1762#endif /* VBOX_STRICT */
1763
1764
1765/**
1766 * Flushes the TLB using EPT.
1767 *
1768 * @returns VBox status code.
1769 * @param pVCpu The cross context virtual CPU structure of the calling
1770 * EMT. Can be NULL depending on @a enmTlbFlush.
1771 * @param enmTlbFlush Type of flush.
1772 *
1773 * @remarks Caller is responsible for making sure this function is called only
1774 * when NestedPaging is supported and providing @a enmTlbFlush that is
1775 * supported by the CPU.
1776 * @remarks Can be called with interrupts disabled.
1777 */
1778static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush)
1779{
1780 uint64_t au64Descriptor[2];
1781 if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
1782 au64Descriptor[0] = 0;
1783 else
1784 {
1785 Assert(pVCpu);
1786 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1787 }
1788 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1789
1790 int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
1791 AssertMsg(rc == VINF_SUCCESS,
1792 ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0, rc));
1793
1794 if ( RT_SUCCESS(rc)
1795 && pVCpu)
1796 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1797}
1798
1799
1800/**
1801 * Flushes the TLB using VPID.
1802 *
1803 * @returns VBox status code.
1804 * @param pVCpu The cross context virtual CPU structure of the calling
1805 * EMT. Can be NULL depending on @a enmTlbFlush.
1806 * @param enmTlbFlush Type of flush.
1807 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1808 * on @a enmTlbFlush).
1809 *
1810 * @remarks Can be called with interrupts disabled.
1811 */
1812static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
1813{
1814 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
1815
1816 uint64_t au64Descriptor[2];
1817 if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
1818 {
1819 au64Descriptor[0] = 0;
1820 au64Descriptor[1] = 0;
1821 }
1822 else
1823 {
1824 AssertPtr(pVCpu);
1825 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1826 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1827 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1828 au64Descriptor[1] = GCPtr;
1829 }
1830
1831 int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
1832 AssertMsg(rc == VINF_SUCCESS,
1833 ("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1834
1835 if ( RT_SUCCESS(rc)
1836 && pVCpu)
1837 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1838 NOREF(rc);
1839}
1840
1841
1842/**
1843 * Invalidates a guest page by guest virtual address. Only relevant for
1844 * EPT/VPID, otherwise there is nothing really to invalidate.
1845 *
1846 * @returns VBox status code.
1847 * @param pVCpu The cross context virtual CPU structure.
1848 * @param GCVirt Guest virtual address of the page to invalidate.
1849 */
1850VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
1851{
1852 AssertPtr(pVCpu);
1853 LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
1854
1855 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1856 if (!fFlushPending)
1857 {
1858 /*
1859 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for
1860 * the EPT case. See @bugref{6043} and @bugref{6177}.
1861 *
1862 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
1863 * as this function maybe called in a loop with individual addresses.
1864 */
1865 PVM pVM = pVCpu->CTX_SUFF(pVM);
1866 if (pVM->hm.s.vmx.fVpid)
1867 {
1868 bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1869
1870#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1871 /*
1872 * Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1873 * where executing INVVPID outside 64-bit mode does not flush translations of
1874 * 64-bit linear addresses, see @bugref{6208#c72}.
1875 */
1876 if (RT_HI_U32(GCVirt))
1877 fVpidFlush = false;
1878#endif
1879
1880 if (fVpidFlush)
1881 {
1882 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
1883 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1884 }
1885 else
1886 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1887 }
1888 else if (pVM->hm.s.fNestedPaging)
1889 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1890 }
1891
1892 return VINF_SUCCESS;
1893}
1894
1895
1896/**
1897 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1898 * case where neither EPT nor VPID is supported by the CPU.
1899 *
1900 * @param pVCpu The cross context virtual CPU structure.
1901 * @param pCpu Pointer to the global HM struct.
1902 *
1903 * @remarks Called with interrupts disabled.
1904 */
1905static void hmR0VmxFlushTaggedTlbNone(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1906{
1907 AssertPtr(pVCpu);
1908 AssertPtr(pCpu);
1909
1910 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1911
1912 Assert(pCpu->idCpu != NIL_RTCPUID);
1913 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1914 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1915 pVCpu->hm.s.fForceTLBFlush = false;
1916 return;
1917}
1918
1919
1920/**
1921 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1922 *
1923 * @param pVCpu The cross context virtual CPU structure.
1924 * @param pCpu Pointer to the global HM CPU struct.
1925 *
1926 * @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
1927 * nomenclature. The reason is, to avoid confusion in compare statements
1928 * since the host-CPU copies are named "ASID".
1929 *
1930 * @remarks Called with interrupts disabled.
1931 */
1932static void hmR0VmxFlushTaggedTlbBoth(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1933{
1934#ifdef VBOX_WITH_STATISTICS
1935 bool fTlbFlushed = false;
1936# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1937# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1938 if (!fTlbFlushed) \
1939 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1940 } while (0)
1941#else
1942# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1943# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1944#endif
1945
1946 AssertPtr(pCpu);
1947 AssertPtr(pVCpu);
1948 Assert(pCpu->idCpu != NIL_RTCPUID);
1949
1950 PVM pVM = pVCpu->CTX_SUFF(pVM);
1951 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1952 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1953 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1954
1955 /*
1956 * Force a TLB flush for the first world-switch if the current CPU differs from the one we
1957 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1958 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1959 * cannot reuse the current ASID anymore.
1960 */
1961 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1962 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1963 {
1964 ++pCpu->uCurrentAsid;
1965 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1966 {
1967 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1968 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1969 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1970 }
1971
1972 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1973 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1974 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1975
1976 /*
1977 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1978 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1979 */
1980 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
1981 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1982 HMVMX_SET_TAGGED_TLB_FLUSHED();
1983 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1984 }
1985 else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
1986 {
1987 /*
1988 * Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
1989 * creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
1990 * tables when EPT is in use. Flushing-by-VPID will only flush linear (only
1991 * VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
1992 * mappings, see @bugref{6568}.
1993 *
1994 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
1995 */
1996 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
1997 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1998 HMVMX_SET_TAGGED_TLB_FLUSHED();
1999 }
2000
2001 pVCpu->hm.s.fForceTLBFlush = false;
2002 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2003
2004 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2005 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2006 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2007 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2008 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2009 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2010 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2011 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2012 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2013
2014 /* Update VMCS with the VPID. */
2015 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2016 AssertRC(rc);
2017
2018#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2019}
2020
2021
2022/**
2023 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2024 *
2025 * @returns VBox status code.
2026 * @param pVCpu The cross context virtual CPU structure.
2027 * @param pCpu Pointer to the global HM CPU struct.
2028 *
2029 * @remarks Called with interrupts disabled.
2030 */
2031static void hmR0VmxFlushTaggedTlbEpt(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2032{
2033 AssertPtr(pVCpu);
2034 AssertPtr(pCpu);
2035 Assert(pCpu->idCpu != NIL_RTCPUID);
2036 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2037 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2038
2039 /*
2040 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2041 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2042 */
2043 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2044 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2045 {
2046 pVCpu->hm.s.fForceTLBFlush = true;
2047 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2048 }
2049
2050 /* Check for explicit TLB flushes. */
2051 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2052 {
2053 pVCpu->hm.s.fForceTLBFlush = true;
2054 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2055 }
2056
2057 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2058 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2059
2060 if (pVCpu->hm.s.fForceTLBFlush)
2061 {
2062 hmR0VmxFlushEpt(pVCpu, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2063 pVCpu->hm.s.fForceTLBFlush = false;
2064 }
2065}
2066
2067
2068/**
2069 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2070 *
2071 * @returns VBox status code.
2072 * @param pVCpu The cross context virtual CPU structure.
2073 * @param pCpu Pointer to the global HM CPU struct.
2074 *
2075 * @remarks Called with interrupts disabled.
2076 */
2077static void hmR0VmxFlushTaggedTlbVpid(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2078{
2079 AssertPtr(pVCpu);
2080 AssertPtr(pCpu);
2081 Assert(pCpu->idCpu != NIL_RTCPUID);
2082 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2083 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2084
2085 /*
2086 * Force a TLB flush for the first world switch if the current CPU differs from the one we
2087 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2088 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2089 * cannot reuse the current ASID anymore.
2090 */
2091 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2092 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2093 {
2094 pVCpu->hm.s.fForceTLBFlush = true;
2095 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2096 }
2097
2098 /* Check for explicit TLB flushes. */
2099 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2100 {
2101 /*
2102 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2103 * hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2104 * fExplicitFlush = true here and change the pCpu->fFlushAsidBeforeUse check below to
2105 * include fExplicitFlush's too) - an obscure corner case.
2106 */
2107 pVCpu->hm.s.fForceTLBFlush = true;
2108 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2109 }
2110
2111 PVM pVM = pVCpu->CTX_SUFF(pVM);
2112 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2113 if (pVCpu->hm.s.fForceTLBFlush)
2114 {
2115 ++pCpu->uCurrentAsid;
2116 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2117 {
2118 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2119 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2120 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2121 }
2122
2123 pVCpu->hm.s.fForceTLBFlush = false;
2124 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2125 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2126 if (pCpu->fFlushAsidBeforeUse)
2127 {
2128 if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2129 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2130 else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2131 {
2132 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2133 pCpu->fFlushAsidBeforeUse = false;
2134 }
2135 else
2136 {
2137 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2138 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2139 }
2140 }
2141 }
2142
2143 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2144 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2145 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2146 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2147 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2148 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2149 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2150
2151 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2152 AssertRC(rc);
2153}
2154
2155
2156/**
2157 * Flushes the guest TLB entry based on CPU capabilities.
2158 *
2159 * @param pVCpu The cross context virtual CPU structure.
2160 * @param pCpu Pointer to the global HM CPU struct.
2161 */
2162DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2163{
2164#ifdef HMVMX_ALWAYS_FLUSH_TLB
2165 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2166#endif
2167 PVM pVM = pVCpu->CTX_SUFF(pVM);
2168 switch (pVM->hm.s.vmx.enmTlbFlushType)
2169 {
2170 case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVCpu, pCpu); break;
2171 case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pVCpu, pCpu); break;
2172 case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pVCpu, pCpu); break;
2173 case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pVCpu, pCpu); break;
2174 default:
2175 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2176 break;
2177 }
2178 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2179}
2180
2181
2182/**
2183 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2184 * TLB entries from the host TLB before VM-entry.
2185 *
2186 * @returns VBox status code.
2187 * @param pVM The cross context VM structure.
2188 */
2189static int hmR0VmxSetupTaggedTlb(PVM pVM)
2190{
2191 /*
2192 * Determine optimal flush type for Nested Paging.
2193 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2194 * guest execution (see hmR3InitFinalizeR0()).
2195 */
2196 if (pVM->hm.s.fNestedPaging)
2197 {
2198 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2199 {
2200 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2201 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2202 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2203 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2204 else
2205 {
2206 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2207 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2208 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2209 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2210 }
2211
2212 /* Make sure the write-back cacheable memory type for EPT is supported. */
2213 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2214 {
2215 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2216 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2217 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2218 }
2219
2220 /* EPT requires a page-walk length of 4. */
2221 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2222 {
2223 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2224 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2225 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2226 }
2227 }
2228 else
2229 {
2230 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2231 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2232 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2233 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2234 }
2235 }
2236
2237 /*
2238 * Determine optimal flush type for VPID.
2239 */
2240 if (pVM->hm.s.vmx.fVpid)
2241 {
2242 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2243 {
2244 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2245 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2246 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2247 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2248 else
2249 {
2250 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2251 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2252 LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2253 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2254 LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2255 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2256 pVM->hm.s.vmx.fVpid = false;
2257 }
2258 }
2259 else
2260 {
2261 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2262 Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2263 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2264 pVM->hm.s.vmx.fVpid = false;
2265 }
2266 }
2267
2268 /*
2269 * Setup the handler for flushing tagged-TLBs.
2270 */
2271 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2272 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2273 else if (pVM->hm.s.fNestedPaging)
2274 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2275 else if (pVM->hm.s.vmx.fVpid)
2276 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2277 else
2278 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2279 return VINF_SUCCESS;
2280}
2281
2282
2283/**
2284 * Sets up pin-based VM-execution controls in the VMCS.
2285 *
2286 * @returns VBox status code.
2287 * @param pVCpu The cross context virtual CPU structure.
2288 *
2289 * @remarks We don't really care about optimizing vmwrites here as it's done only
2290 * once per VM and hence we don't care about VMCS-field cache comparisons.
2291 */
2292static int hmR0VmxSetupPinCtls(PVMCPU pVCpu)
2293{
2294 PVM pVM = pVCpu->CTX_SUFF(pVM);
2295 uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.disallowed0; /* Bits set here must always be set. */
2296 uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2297
2298 fVal |= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2299 | VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2300
2301 if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2302 fVal |= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2303
2304 /* Enable the VMX preemption timer. */
2305 if (pVM->hm.s.vmx.fUsePreemptTimer)
2306 {
2307 Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2308 fVal |= VMX_PIN_CTLS_PREEMPT_TIMER;
2309 }
2310
2311#if 0
2312 /* Enable posted-interrupt processing. */
2313 if (pVM->hm.s.fPostedIntrs)
2314 {
2315 Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
2316 Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
2317 fVal |= VMX_PIN_CTL_POSTED_INT;
2318 }
2319#endif
2320
2321 if ((fVal & fZap) != fVal)
2322 {
2323 LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2324 pVM->hm.s.vmx.Msrs.PinCtls.n.disallowed0, fVal, fZap));
2325 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2326 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2327 }
2328
2329 /* Commit it to the VMCS and update our cache. */
2330 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2331 AssertRCReturn(rc, rc);
2332 pVCpu->hm.s.vmx.u32PinCtls = fVal;
2333
2334 return VINF_SUCCESS;
2335}
2336
2337
2338/**
2339 * Sets up secondary processor-based VM-execution controls in the VMCS.
2340 *
2341 * @returns VBox status code.
2342 * @param pVCpu The cross context virtual CPU structure.
2343 *
2344 * @remarks We don't really care about optimizing vmwrites here as it's done only
2345 * once per VM and hence we don't care about VMCS-field cache comparisons.
2346 */
2347static int hmR0VmxSetupProcCtls2(PVMCPU pVCpu)
2348{
2349 PVM pVM = pVCpu->CTX_SUFF(pVM);
2350 uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2351 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2352
2353 /* WBINVD causes a VM-exit. */
2354 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
2355 fVal |= VMX_PROC_CTLS2_WBINVD_EXIT;
2356
2357 /* Enable EPT (aka nested-paging). */
2358 if (pVM->hm.s.fNestedPaging)
2359 fVal |= VMX_PROC_CTLS2_EPT;
2360
2361 /*
2362 * Enable the INVPCID instruction if supported by the hardware and we expose
2363 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2364 */
2365 if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID)
2366 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2367 fVal |= VMX_PROC_CTLS2_INVPCID;
2368
2369 /* Enable VPID. */
2370 if (pVM->hm.s.vmx.fVpid)
2371 fVal |= VMX_PROC_CTLS2_VPID;
2372
2373 /* Enable Unrestricted guest execution. */
2374 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2375 fVal |= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
2376
2377#if 0
2378 if (pVM->hm.s.fVirtApicRegs)
2379 {
2380 /* Enable APIC-register virtualization. */
2381 Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
2382 fVal |= VMX_PROC_CTLS2_APIC_REG_VIRT;
2383
2384 /* Enable virtual-interrupt delivery. */
2385 Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
2386 fVal |= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
2387 }
2388#endif
2389
2390 /* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is where the TPR shadow resides. */
2391 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2392 * done dynamically. */
2393 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
2394 {
2395 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2396 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2397 fVal |= VMX_PROC_CTLS2_VIRT_APIC_ACCESS; /* Virtualize APIC accesses. */
2398 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2399 AssertRCReturn(rc, rc);
2400 }
2401
2402 /* Enable RDTSCP. */
2403 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP)
2404 fVal |= VMX_PROC_CTLS2_RDTSCP;
2405
2406 /* Enable Pause-Loop exiting. */
2407 if ( pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT
2408 && pVM->hm.s.vmx.cPleGapTicks
2409 && pVM->hm.s.vmx.cPleWindowTicks)
2410 {
2411 fVal |= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
2412
2413 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2414 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2415 AssertRCReturn(rc, rc);
2416 }
2417
2418 if ((fVal & fZap) != fVal)
2419 {
2420 LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2421 pVM->hm.s.vmx.Msrs.ProcCtls2.n.disallowed0, fVal, fZap));
2422 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2423 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2424 }
2425
2426 /* Commit it to the VMCS and update our cache. */
2427 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2428 AssertRCReturn(rc, rc);
2429 pVCpu->hm.s.vmx.u32ProcCtls2 = fVal;
2430
2431 return VINF_SUCCESS;
2432}
2433
2434
2435/**
2436 * Sets up processor-based VM-execution controls in the VMCS.
2437 *
2438 * @returns VBox status code.
2439 * @param pVCpu The cross context virtual CPU structure.
2440 *
2441 * @remarks We don't really care about optimizing vmwrites here as it's done only
2442 * once per VM and hence we don't care about VMCS-field cache comparisons.
2443 */
2444static int hmR0VmxSetupProcCtls(PVMCPU pVCpu)
2445{
2446 PVM pVM = pVCpu->CTX_SUFF(pVM);
2447 uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2448 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2449
2450 fVal |= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
2451 | VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2452 | VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2453 | VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2454 | VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2455 | VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2456 | VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2457
2458 /* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2459 if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
2460 || (pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
2461 {
2462 LogRelFunc(("Unsupported VMX_PROC_CTLS_MOV_DR_EXIT combo!"));
2463 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2464 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2465 }
2466
2467 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2468 if (!pVM->hm.s.fNestedPaging)
2469 {
2470 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2471 fVal |= VMX_PROC_CTLS_INVLPG_EXIT
2472 | VMX_PROC_CTLS_CR3_LOAD_EXIT
2473 | VMX_PROC_CTLS_CR3_STORE_EXIT;
2474 }
2475
2476 /* Use TPR shadowing if supported by the CPU. */
2477 if ( PDMHasApic(pVM)
2478 && pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
2479 {
2480 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2481 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2482 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2483 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2484 AssertRCReturn(rc, rc);
2485
2486 fVal |= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2487 /* CR8 writes cause a VM-exit based on TPR threshold. */
2488 Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
2489 Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
2490 }
2491 else
2492 {
2493 /*
2494 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2495 * Set this control only for 64-bit guests.
2496 */
2497 if (pVM->hm.s.fAllow64BitGuests)
2498 {
2499 fVal |= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2500 | VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2501 }
2502 }
2503
2504 /* Use MSR-bitmaps if supported by the CPU. */
2505 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2506 {
2507 fVal |= VMX_PROC_CTLS_USE_MSR_BITMAPS;
2508
2509 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2510 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2511 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2512 AssertRCReturn(rc, rc);
2513
2514 /*
2515 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2516 * automatically using dedicated fields in the VMCS.
2517 */
2518 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2519 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2520 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2521 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2522 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2523#if HC_ARCH_BITS == 64
2524 /*
2525 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2526 */
2527 if (pVM->hm.s.fAllow64BitGuests)
2528 {
2529 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2530 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2531 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2532 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2533 }
2534#endif
2535 /*
2536 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2537 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2538 */
2539 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2540 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2541
2542 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2543 }
2544
2545 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2546 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2547 fVal |= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
2548
2549 if ((fVal & fZap) != fVal)
2550 {
2551 LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2552 pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0, fVal, fZap));
2553 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2554 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2555 }
2556
2557 /* Commit it to the VMCS and update our cache. */
2558 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2559 AssertRCReturn(rc, rc);
2560 pVCpu->hm.s.vmx.u32ProcCtls = fVal;
2561
2562 /* Set up secondary processor-based VM-execution controls if the CPU supports it. */
2563 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2564 return hmR0VmxSetupProcCtls2(pVCpu);
2565
2566 /* Sanity check, should not really happen. */
2567 if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2568 {
2569 LogRelFunc(("Unrestricted Guest enabled when secondary processor-based VM-execution controls not available\n"));
2570 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2571 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2572 }
2573
2574 /* Old CPUs without secondary processor-based VM-execution controls would end up here. */
2575 return VINF_SUCCESS;
2576}
2577
2578
2579/**
2580 * Sets up miscellaneous (everything other than Pin & Processor-based
2581 * VM-execution) control fields in the VMCS.
2582 *
2583 * @returns VBox status code.
2584 * @param pVCpu The cross context virtual CPU structure.
2585 */
2586static int hmR0VmxSetupMiscCtls(PVMCPU pVCpu)
2587{
2588 AssertPtr(pVCpu);
2589
2590 int rc = VERR_GENERAL_FAILURE;
2591
2592 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2593#if 0
2594 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxExportGuestCR3AndCR4())*/
2595 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2596 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2597
2598 /*
2599 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2600 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2601 * We thus use the exception bitmap to control it rather than use both.
2602 */
2603 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2604 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2605
2606 /* All IO & IOIO instructions cause VM-exits. */
2607 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2608 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2609
2610 /* Initialize the MSR-bitmap area. */
2611 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2612 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2613 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2614 AssertRCReturn(rc, rc);
2615#endif
2616
2617 /* Setup MSR auto-load/store area. */
2618 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2619 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2620 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2621 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2622 AssertRCReturn(rc, rc);
2623
2624 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2625 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2626 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2627 AssertRCReturn(rc, rc);
2628
2629 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2630 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2631 AssertRCReturn(rc, rc);
2632
2633 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2634#if 0
2635 /* Setup debug controls */
2636 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2637 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2638 AssertRCReturn(rc, rc);
2639#endif
2640
2641 return rc;
2642}
2643
2644
2645/**
2646 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2647 *
2648 * We shall setup those exception intercepts that don't change during the
2649 * lifetime of the VM here. The rest are done dynamically while loading the
2650 * guest state.
2651 *
2652 * @returns VBox status code.
2653 * @param pVCpu The cross context virtual CPU structure.
2654 */
2655static int hmR0VmxInitXcptBitmap(PVMCPU pVCpu)
2656{
2657 AssertPtr(pVCpu);
2658
2659 uint32_t uXcptBitmap;
2660
2661 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2662 uXcptBitmap = RT_BIT_32(X86_XCPT_AC);
2663
2664 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2665 and writes, and because recursive #DBs can cause the CPU hang, we must always
2666 intercept #DB. */
2667 uXcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2668
2669 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2670 if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
2671 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
2672
2673 /* Commit it to the VMCS. */
2674 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
2675 AssertRCReturn(rc, rc);
2676
2677 /* Update our cache of the exception bitmap. */
2678 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
2679 return VINF_SUCCESS;
2680}
2681
2682
2683/**
2684 * Does per-VM VT-x initialization.
2685 *
2686 * @returns VBox status code.
2687 * @param pVM The cross context VM structure.
2688 */
2689VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2690{
2691 LogFlowFunc(("pVM=%p\n", pVM));
2692
2693 int rc = hmR0VmxStructsAlloc(pVM);
2694 if (RT_FAILURE(rc))
2695 {
2696 LogRelFunc(("hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2697 return rc;
2698 }
2699
2700 return VINF_SUCCESS;
2701}
2702
2703
2704/**
2705 * Does per-VM VT-x termination.
2706 *
2707 * @returns VBox status code.
2708 * @param pVM The cross context VM structure.
2709 */
2710VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2711{
2712 LogFlowFunc(("pVM=%p\n", pVM));
2713
2714#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2715 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2716 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2717#endif
2718 hmR0VmxStructsFree(pVM);
2719 return VINF_SUCCESS;
2720}
2721
2722
2723/**
2724 * Sets up the VM for execution under VT-x.
2725 * This function is only called once per-VM during initialization.
2726 *
2727 * @returns VBox status code.
2728 * @param pVM The cross context VM structure.
2729 */
2730VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2731{
2732 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2733 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2734
2735 LogFlowFunc(("pVM=%p\n", pVM));
2736
2737 /*
2738 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be
2739 * allocated. We no longer support the highly unlikely case of UnrestrictedGuest without
2740 * pRealModeTSS, see hmR3InitFinalizeR0Intel().
2741 */
2742 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2743 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2744 || !pVM->hm.s.vmx.pRealModeTSS))
2745 {
2746 LogRelFunc(("Invalid real-on-v86 state.\n"));
2747 return VERR_INTERNAL_ERROR;
2748 }
2749
2750 /* Initialize these always, see hmR3InitFinalizeR0().*/
2751 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
2752 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
2753
2754 /* Setup the tagged-TLB flush handlers. */
2755 int rc = hmR0VmxSetupTaggedTlb(pVM);
2756 if (RT_FAILURE(rc))
2757 {
2758 LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2759 return rc;
2760 }
2761
2762 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2763 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2764#if HC_ARCH_BITS == 64
2765 if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
2766 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2767 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR))
2768 {
2769 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2770 }
2771#endif
2772
2773 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2774 RTCCUINTREG uHostCR4 = ASMGetCR4();
2775 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2776 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2777
2778 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2779 {
2780 PVMCPU pVCpu = &pVM->aCpus[i];
2781 AssertPtr(pVCpu);
2782 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2783
2784 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2785 Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2786
2787 /* Set revision dword at the beginning of the VMCS structure. */
2788 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
2789
2790 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2791 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2792 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc\n", rc),
2793 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2794
2795 /* Load this VMCS as the current VMCS. */
2796 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2797 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc\n", rc),
2798 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2799
2800 rc = hmR0VmxSetupPinCtls(pVCpu);
2801 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc\n", rc),
2802 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2803
2804 rc = hmR0VmxSetupProcCtls(pVCpu);
2805 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc\n", rc),
2806 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2807
2808 rc = hmR0VmxSetupMiscCtls(pVCpu);
2809 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc\n", rc),
2810 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2811
2812 rc = hmR0VmxInitXcptBitmap(pVCpu);
2813 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc\n", rc),
2814 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2815
2816#if HC_ARCH_BITS == 32
2817 rc = hmR0VmxInitVmcsReadCache(pVCpu);
2818 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc\n", rc),
2819 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2820#endif
2821
2822 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2823 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2824 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc\n", rc),
2825 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2826
2827 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2828
2829 hmR0VmxUpdateErrorRecord(pVCpu, rc);
2830 }
2831
2832 return VINF_SUCCESS;
2833}
2834
2835
2836/**
2837 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2838 * the VMCS.
2839 *
2840 * @returns VBox status code.
2841 */
2842static int hmR0VmxExportHostControlRegs(void)
2843{
2844 RTCCUINTREG uReg = ASMGetCR0();
2845 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2846 AssertRCReturn(rc, rc);
2847
2848 uReg = ASMGetCR3();
2849 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2850 AssertRCReturn(rc, rc);
2851
2852 uReg = ASMGetCR4();
2853 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2854 AssertRCReturn(rc, rc);
2855 return rc;
2856}
2857
2858
2859/**
2860 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2861 * the host-state area in the VMCS.
2862 *
2863 * @returns VBox status code.
2864 * @param pVCpu The cross context virtual CPU structure.
2865 */
2866static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
2867{
2868#if HC_ARCH_BITS == 64
2869/**
2870 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2871 * requirements. See hmR0VmxExportHostSegmentRegs().
2872 */
2873# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2874 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2875 { \
2876 bool fValidSelector = true; \
2877 if ((selValue) & X86_SEL_LDT) \
2878 { \
2879 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2880 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2881 } \
2882 if (fValidSelector) \
2883 { \
2884 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2885 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2886 } \
2887 (selValue) = 0; \
2888 }
2889
2890 /*
2891 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2892 * should -not- save the messed up state without restoring the original host-state,
2893 * see @bugref{7240}.
2894 *
2895 * This apparently can happen (most likely the FPU changes), deal with it rather than
2896 * asserting. Was observed booting Solaris 10u10 32-bit guest.
2897 */
2898 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2899 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2900 {
2901 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2902 pVCpu->idCpu));
2903 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2904 }
2905 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2906#else
2907 RT_NOREF(pVCpu);
2908#endif
2909
2910 /*
2911 * Host DS, ES, FS and GS segment registers.
2912 */
2913#if HC_ARCH_BITS == 64
2914 RTSEL uSelDS = ASMGetDS();
2915 RTSEL uSelES = ASMGetES();
2916 RTSEL uSelFS = ASMGetFS();
2917 RTSEL uSelGS = ASMGetGS();
2918#else
2919 RTSEL uSelDS = 0;
2920 RTSEL uSelES = 0;
2921 RTSEL uSelFS = 0;
2922 RTSEL uSelGS = 0;
2923#endif
2924
2925 /*
2926 * Host CS and SS segment registers.
2927 */
2928 RTSEL uSelCS = ASMGetCS();
2929 RTSEL uSelSS = ASMGetSS();
2930
2931 /*
2932 * Host TR segment register.
2933 */
2934 RTSEL uSelTR = ASMGetTR();
2935
2936#if HC_ARCH_BITS == 64
2937 /*
2938 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
2939 * gain VM-entry and restore them before we get preempted.
2940 *
2941 * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2942 */
2943 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2944 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2945 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2946 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2947# undef VMXLOCAL_ADJUST_HOST_SEG
2948#endif
2949
2950 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2951 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2952 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2953 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2954 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2955 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2956 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2957 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2958 Assert(uSelCS);
2959 Assert(uSelTR);
2960
2961 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2962#if 0
2963 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE))
2964 Assert(uSelSS != 0);
2965#endif
2966
2967 /* Write these host selector fields into the host-state area in the VMCS. */
2968 int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2969 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2970#if HC_ARCH_BITS == 64
2971 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2972 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2973 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2974 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2975#else
2976 NOREF(uSelDS);
2977 NOREF(uSelES);
2978 NOREF(uSelFS);
2979 NOREF(uSelGS);
2980#endif
2981 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2982 AssertRCReturn(rc, rc);
2983
2984 /*
2985 * Host GDTR and IDTR.
2986 */
2987 RTGDTR Gdtr;
2988 RTIDTR Idtr;
2989 RT_ZERO(Gdtr);
2990 RT_ZERO(Idtr);
2991 ASMGetGDTR(&Gdtr);
2992 ASMGetIDTR(&Idtr);
2993 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2994 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2995 AssertRCReturn(rc, rc);
2996
2997#if HC_ARCH_BITS == 64
2998 /*
2999 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
3000 * them to the maximum limit (0xffff) on every VM-exit.
3001 */
3002 if (Gdtr.cbGdt != 0xffff)
3003 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3004
3005 /*
3006 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
3007 * Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
3008 * as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
3009 * However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
3010 * uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
3011 * alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
3012 * at 0xffff on hosts where we are sure it won't cause trouble.
3013 */
3014# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3015 if (Idtr.cbIdt < 0x0fff)
3016# else
3017 if (Idtr.cbIdt != 0xffff)
3018# endif
3019 {
3020 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3021 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3022 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3023 }
3024#endif
3025
3026 /*
3027 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3028 * and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3029 * RPL should be too in most cases.
3030 */
3031 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3032 ("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
3033
3034 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3035#if HC_ARCH_BITS == 64
3036 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3037
3038 /*
3039 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3040 * all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3041 * restoration if the host has something else. Task switching is not supported in 64-bit
3042 * mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3043 * limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3044 *
3045 * [1] See Intel spec. 3.5 "System Descriptor Types".
3046 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3047 */
3048 PVM pVM = pVCpu->CTX_SUFF(pVM);
3049 Assert(pDesc->System.u4Type == 11);
3050 if ( pDesc->System.u16LimitLow != 0x67
3051 || pDesc->System.u4LimitHigh)
3052 {
3053 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3054 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3055 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3056 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3057 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3058 }
3059
3060 /*
3061 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3062 */
3063 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3064 {
3065 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3066 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3067 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3068 {
3069 /* The GDT is read-only but the writable GDT is available. */
3070 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3071 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3072 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3073 AssertRCReturn(rc, rc);
3074 }
3075 }
3076#else
3077 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3078#endif
3079 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3080 AssertRCReturn(rc, rc);
3081
3082 /*
3083 * Host FS base and GS base.
3084 */
3085#if HC_ARCH_BITS == 64
3086 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3087 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3088 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3089 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3090 AssertRCReturn(rc, rc);
3091
3092 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3093 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3094 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3095 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3096 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3097#endif
3098 return VINF_SUCCESS;
3099}
3100
3101
3102/**
3103 * Exports certain host MSRs in the VM-exit MSR-load area and some in the
3104 * host-state area of the VMCS.
3105 *
3106 * Theses MSRs will be automatically restored on the host after every successful
3107 * VM-exit.
3108 *
3109 * @returns VBox status code.
3110 * @param pVCpu The cross context virtual CPU structure.
3111 *
3112 * @remarks No-long-jump zone!!!
3113 */
3114static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
3115{
3116 AssertPtr(pVCpu);
3117 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3118
3119 /*
3120 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3121 * rather than swapping them on every VM-entry.
3122 */
3123 hmR0VmxLazySaveHostMsrs(pVCpu);
3124
3125 /*
3126 * Host Sysenter MSRs.
3127 */
3128 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3129#if HC_ARCH_BITS == 32
3130 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3131 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3132#else
3133 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3134 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3135#endif
3136 AssertRCReturn(rc, rc);
3137
3138 /*
3139 * Host EFER MSR.
3140 *
3141 * If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
3142 * done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
3143 */
3144 PVM pVM = pVCpu->CTX_SUFF(pVM);
3145 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3146 {
3147 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3148 AssertRCReturn(rc, rc);
3149 }
3150
3151 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see hmR0VmxExportGuestExitCtls(). */
3152
3153 return VINF_SUCCESS;
3154}
3155
3156
3157/**
3158 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3159 *
3160 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3161 * these two bits are handled by VM-entry, see hmR0VmxExportGuestExitCtls() and
3162 * hmR0VMxExportGuestEntryCtls().
3163 *
3164 * @returns true if we need to load guest EFER, false otherwise.
3165 * @param pVCpu The cross context virtual CPU structure.
3166 *
3167 * @remarks Requires EFER, CR4.
3168 * @remarks No-long-jump zone!!!
3169 */
3170static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu)
3171{
3172#ifdef HMVMX_ALWAYS_SWAP_EFER
3173 return true;
3174#endif
3175
3176 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3177#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3178 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3179 if (CPUMIsGuestInLongModeEx(pCtx))
3180 return false;
3181#endif
3182
3183 PVM pVM = pVCpu->CTX_SUFF(pVM);
3184 uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3185 uint64_t const u64GuestEfer = pCtx->msrEFER;
3186
3187 /*
3188 * For 64-bit guests, if EFER.SCE bit differs, we need to swap EFER to ensure that the
3189 * guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
3190 */
3191 if ( CPUMIsGuestInLongModeEx(pCtx)
3192 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3193 {
3194 return true;
3195 }
3196
3197 /*
3198 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3199 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3200 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3201 */
3202 if ( (pCtx->cr4 & X86_CR4_PAE)
3203 && (pCtx->cr0 & X86_CR0_PG)
3204 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3205 {
3206 /* Assert that host is NX capable. */
3207 Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
3208 return true;
3209 }
3210
3211 return false;
3212}
3213
3214
3215/**
3216 * Exports the guest state with appropriate VM-entry controls in the VMCS.
3217 *
3218 * These controls can affect things done on VM-exit; e.g. "load debug controls",
3219 * see Intel spec. 24.8.1 "VM-entry controls".
3220 *
3221 * @returns VBox status code.
3222 * @param pVCpu The cross context virtual CPU structure.
3223 *
3224 * @remarks Requires EFER.
3225 * @remarks No-long-jump zone!!!
3226 */
3227static int hmR0VmxExportGuestEntryCtls(PVMCPU pVCpu)
3228{
3229 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_CTLS)
3230 {
3231 PVM pVM = pVCpu->CTX_SUFF(pVM);
3232 uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
3233 uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3234
3235 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3236 fVal |= VMX_ENTRY_CTLS_LOAD_DEBUG;
3237
3238 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3239 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
3240 {
3241 fVal |= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
3242 Log4Func(("VMX_ENTRY_CTLS_IA32E_MODE_GUEST\n"));
3243 }
3244 else
3245 Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
3246
3247 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3248 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3249 && hmR0VmxShouldSwapEferMsr(pVCpu))
3250 {
3251 fVal |= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
3252 Log4Func(("VMX_ENTRY_CTLS_LOAD_EFER_MSR\n"));
3253 }
3254
3255 /*
3256 * The following should -not- be set (since we're not in SMM mode):
3257 * - VMX_ENTRY_CTLS_ENTRY_TO_SMM
3258 * - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
3259 */
3260
3261 /** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
3262 * VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
3263
3264 if ((fVal & fZap) != fVal)
3265 {
3266 Log4Func(("Invalid VM-entry controls combo! Cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3267 pVM->hm.s.vmx.Msrs.EntryCtls.n.disallowed0, fVal, fZap));
3268 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3269 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3270 }
3271
3272 /* Commit it to the VMCS and update our cache. */
3273 if (pVCpu->hm.s.vmx.u32EntryCtls != fVal)
3274 {
3275 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
3276 AssertRCReturn(rc, rc);
3277 pVCpu->hm.s.vmx.u32EntryCtls = fVal;
3278 }
3279
3280 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_CTLS);
3281 }
3282 return VINF_SUCCESS;
3283}
3284
3285
3286/**
3287 * Exports the guest state with appropriate VM-exit controls in the VMCS.
3288 *
3289 * @returns VBox status code.
3290 * @param pVCpu The cross context virtual CPU structure.
3291 *
3292 * @remarks Requires EFER.
3293 */
3294static int hmR0VmxExportGuestExitCtls(PVMCPU pVCpu)
3295{
3296 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_EXIT_CTLS)
3297 {
3298 PVM pVM = pVCpu->CTX_SUFF(pVM);
3299 uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
3300 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3301
3302 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3303 fVal |= VMX_EXIT_CTLS_SAVE_DEBUG;
3304
3305 /*
3306 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3307 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in
3308 * hmR0VmxExportHostMsrs().
3309 */
3310#if HC_ARCH_BITS == 64
3311 fVal |= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
3312 Log4Func(("VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE\n"));
3313#else
3314 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3315 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3316 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3317 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3318 {
3319 /* The switcher returns to long mode, EFER is managed by the switcher. */
3320 fVal |= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
3321 Log4Func(("VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE\n"));
3322 }
3323 else
3324 Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
3325#endif
3326
3327 /* If the newer VMCS fields for managing EFER exists, use it. */
3328 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3329 && hmR0VmxShouldSwapEferMsr(pVCpu))
3330 {
3331 fVal |= VMX_EXIT_CTLS_SAVE_EFER_MSR
3332 | VMX_EXIT_CTLS_LOAD_EFER_MSR;
3333 Log4Func(("VMX_EXIT_CTLS_SAVE_EFER_MSR and VMX_EXIT_CTLS_LOAD_EFER_MSR\n"));
3334 }
3335
3336 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3337 Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
3338
3339 /** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
3340 * VMX_EXIT_CTLS_SAVE_PAT_MSR,
3341 * VMX_EXIT_CTLS_LOAD_PAT_MSR. */
3342
3343 /* Enable saving of the VMX preemption timer value on VM-exit. */
3344 if ( pVM->hm.s.vmx.fUsePreemptTimer
3345 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_VMX_PREEMPT_TIMER))
3346 fVal |= VMX_EXIT_CTLS_SAVE_VMX_PREEMPT_TIMER;
3347
3348 if ((fVal & fZap) != fVal)
3349 {
3350 LogRelFunc(("Invalid VM-exit controls combo! cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3351 pVM->hm.s.vmx.Msrs.ExitCtls.n.disallowed0, fVal, fZap));
3352 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3353 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3354 }
3355
3356 /* Commit it to the VMCS and update our cache. */
3357 if (pVCpu->hm.s.vmx.u32ExitCtls != fVal)
3358 {
3359 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
3360 AssertRCReturn(rc, rc);
3361 pVCpu->hm.s.vmx.u32ExitCtls = fVal;
3362 }
3363
3364 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_EXIT_CTLS);
3365 }
3366 return VINF_SUCCESS;
3367}
3368
3369
3370/**
3371 * Sets the TPR threshold in the VMCS.
3372 *
3373 * @returns VBox status code.
3374 * @param pVCpu The cross context virtual CPU structure.
3375 * @param u32TprThreshold The TPR threshold (task-priority class only).
3376 */
3377DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3378{
3379 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3380 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3381 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3382}
3383
3384
3385/**
3386 * Exports the guest APIC TPR state into the VMCS.
3387 *
3388 * @returns VBox status code.
3389 * @param pVCpu The cross context virtual CPU structure.
3390 *
3391 * @remarks No-long-jump zone!!!
3392 */
3393static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu)
3394{
3395 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
3396 {
3397 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
3398
3399 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3400 && APICIsEnabled(pVCpu))
3401 {
3402 /*
3403 * Setup TPR shadowing.
3404 */
3405 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
3406 {
3407 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3408
3409 bool fPendingIntr = false;
3410 uint8_t u8Tpr = 0;
3411 uint8_t u8PendingIntr = 0;
3412 int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3413 AssertRCReturn(rc, rc);
3414
3415 /*
3416 * If there are interrupts pending but masked by the TPR, instruct VT-x to
3417 * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
3418 * priority of the pending interrupt so we can deliver the interrupt. If there
3419 * are no interrupts pending, set threshold to 0 to not cause any
3420 * TPR-below-threshold VM-exits.
3421 */
3422 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3423 uint32_t u32TprThreshold = 0;
3424 if (fPendingIntr)
3425 {
3426 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3427 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3428 const uint8_t u8TprPriority = u8Tpr >> 4;
3429 if (u8PendingPriority <= u8TprPriority)
3430 u32TprThreshold = u8PendingPriority;
3431 }
3432
3433 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3434 AssertRCReturn(rc, rc);
3435 }
3436 }
3437 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
3438 }
3439 return VINF_SUCCESS;
3440}
3441
3442
3443/**
3444 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3445 *
3446 * @returns Guest's interruptibility-state.
3447 * @param pVCpu The cross context virtual CPU structure.
3448 *
3449 * @remarks No-long-jump zone!!!
3450 */
3451static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu)
3452{
3453 /*
3454 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3455 */
3456 uint32_t fIntrState = 0;
3457 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3458 {
3459 /* If inhibition is active, RIP & RFLAGS should've been accessed
3460 (i.e. read previously from the VMCS or from ring-3). */
3461 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3462#ifdef VBOX_STRICT
3463 uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
3464 AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
3465#endif
3466 if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3467 {
3468 if (pCtx->eflags.Bits.u1IF)
3469 fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
3470 else
3471 fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
3472 }
3473 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3474 {
3475 /*
3476 * We can clear the inhibit force flag as even if we go back to the recompiler
3477 * without executing guest code in VT-x, the flag's condition to be cleared is
3478 * met and thus the cleared state is correct.
3479 */
3480 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3481 }
3482 }
3483
3484 /*
3485 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3486 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3487 * setting this would block host-NMIs and IRET will not clear the blocking.
3488 *
3489 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3490 */
3491 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3492 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
3493 {
3494 fIntrState |= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
3495 }
3496
3497 return fIntrState;
3498}
3499
3500
3501/**
3502 * Exports the guest's interruptibility-state into the guest-state area in the
3503 * VMCS.
3504 *
3505 * @returns VBox status code.
3506 * @param pVCpu The cross context virtual CPU structure.
3507 * @param fIntrState The interruptibility-state to set.
3508 */
3509static int hmR0VmxExportGuestIntrState(PVMCPU pVCpu, uint32_t fIntrState)
3510{
3511 NOREF(pVCpu);
3512 AssertMsg(!(fIntrState & 0xfffffff0), ("%#x\n", fIntrState)); /* Bits 31:4 MBZ. */
3513 Assert((fIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3514 return VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
3515}
3516
3517
3518/**
3519 * Exports the exception intercepts required for guest execution in the VMCS.
3520 *
3521 * @returns VBox status code.
3522 * @param pVCpu The cross context virtual CPU structure.
3523 *
3524 * @remarks No-long-jump zone!!!
3525 */
3526static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu)
3527{
3528 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
3529 {
3530 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3531
3532 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxExportSharedCR0(). */
3533 if (pVCpu->hm.s.fGIMTrapXcptUD)
3534 uXcptBitmap |= RT_BIT(X86_XCPT_UD);
3535#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3536 else
3537 uXcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3538#endif
3539
3540 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_AC));
3541 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_DB));
3542
3543 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3544 {
3545 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3546 AssertRCReturn(rc, rc);
3547 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3548 }
3549
3550 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3551 Log4Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64\n", uXcptBitmap));
3552 }
3553 return VINF_SUCCESS;
3554}
3555
3556
3557/**
3558 * Exports the guest's RIP into the guest-state area in the VMCS.
3559 *
3560 * @returns VBox status code.
3561 * @param pVCpu The cross context virtual CPU structure.
3562 *
3563 * @remarks No-long-jump zone!!!
3564 */
3565static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
3566{
3567 int rc = VINF_SUCCESS;
3568 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
3569 {
3570 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
3571
3572 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
3573 AssertRCReturn(rc, rc);
3574
3575 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3576 Log4Func(("RIP=%#RX64\n", pVCpu->cpum.GstCtx.rip));
3577 }
3578 return rc;
3579}
3580
3581
3582/**
3583 * Exports the guest's RSP into the guest-state area in the VMCS.
3584 *
3585 * @returns VBox status code.
3586 * @param pVCpu The cross context virtual CPU structure.
3587 *
3588 * @remarks No-long-jump zone!!!
3589 */
3590static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
3591{
3592 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
3593 {
3594 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
3595
3596 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
3597 AssertRCReturn(rc, rc);
3598
3599 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3600 }
3601 return VINF_SUCCESS;
3602}
3603
3604
3605/**
3606 * Exports the guest's RFLAGS into the guest-state area in the VMCS.
3607 *
3608 * @returns VBox status code.
3609 * @param pVCpu The cross context virtual CPU structure.
3610 *
3611 * @remarks No-long-jump zone!!!
3612 */
3613static int hmR0VmxExportGuestRflags(PVMCPU pVCpu)
3614{
3615 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3616 {
3617 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
3618
3619 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3620 Let us assert it as such and use 32-bit VMWRITE. */
3621 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
3622 X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
3623 Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3624 Assert(!(fEFlags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3625
3626 /*
3627 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3628 * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3629 * can run the real-mode guest code under Virtual 8086 mode.
3630 */
3631 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3632 {
3633 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3634 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3635 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3636 fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3637 fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3638 }
3639
3640 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3641 AssertRCReturn(rc, rc);
3642
3643 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3644 Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
3645 }
3646 return VINF_SUCCESS;
3647}
3648
3649
3650/**
3651 * Exports the guest CR0 control register into the guest-state area in the VMCS.
3652 *
3653 * The guest FPU state is always pre-loaded hence we don't need to bother about
3654 * sharing FPU related CR0 bits between the guest and host.
3655 *
3656 * @returns VBox status code.
3657 * @param pVCpu The cross context virtual CPU structure.
3658 *
3659 * @remarks No-long-jump zone!!!
3660 */
3661static int hmR0VmxExportGuestCR0(PVMCPU pVCpu)
3662{
3663 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
3664 {
3665 PVM pVM = pVCpu->CTX_SUFF(pVM);
3666 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
3667 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.cr0));
3668
3669 uint32_t const u32ShadowCr0 = pVCpu->cpum.GstCtx.cr0;
3670 uint32_t u32GuestCr0 = pVCpu->cpum.GstCtx.cr0;
3671
3672 /*
3673 * Setup VT-x's view of the guest CR0.
3674 * Minimize VM-exits due to CR3 changes when we have NestedPaging.
3675 */
3676 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
3677 if (pVM->hm.s.fNestedPaging)
3678 {
3679 if (CPUMIsGuestPagingEnabled(pVCpu))
3680 {
3681 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3682 uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
3683 | VMX_PROC_CTLS_CR3_STORE_EXIT);
3684 }
3685 else
3686 {
3687 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3688 uProcCtls |= VMX_PROC_CTLS_CR3_LOAD_EXIT
3689 | VMX_PROC_CTLS_CR3_STORE_EXIT;
3690 }
3691
3692 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3693 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3694 uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
3695 }
3696 else
3697 {
3698 /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3699 u32GuestCr0 |= X86_CR0_WP;
3700 }
3701
3702 /*
3703 * Guest FPU bits.
3704 *
3705 * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3706 * using CR0.TS.
3707 *
3708 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3709 * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3710 */
3711 u32GuestCr0 |= X86_CR0_NE;
3712
3713 /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3714 bool const fInterceptMF = !(u32ShadowCr0 & X86_CR0_NE);
3715
3716 /*
3717 * Update exception intercepts.
3718 */
3719 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3720 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3721 {
3722 Assert(PDMVmmDevHeapIsEnabled(pVM));
3723 Assert(pVM->hm.s.vmx.pRealModeTSS);
3724 uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3725 }
3726 else
3727 {
3728 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3729 uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3730 if (fInterceptMF)
3731 uXcptBitmap |= RT_BIT(X86_XCPT_MF);
3732 }
3733
3734 /* Additional intercepts for debugging, define these yourself explicitly. */
3735#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3736 uXcptBitmap |= 0
3737 | RT_BIT(X86_XCPT_BP)
3738 | RT_BIT(X86_XCPT_DE)
3739 | RT_BIT(X86_XCPT_NM)
3740 | RT_BIT(X86_XCPT_TS)
3741 | RT_BIT(X86_XCPT_UD)
3742 | RT_BIT(X86_XCPT_NP)
3743 | RT_BIT(X86_XCPT_SS)
3744 | RT_BIT(X86_XCPT_GP)
3745 | RT_BIT(X86_XCPT_PF)
3746 | RT_BIT(X86_XCPT_MF)
3747 ;
3748#elif defined(HMVMX_ALWAYS_TRAP_PF)
3749 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
3750#endif
3751 Assert(pVM->hm.s.fNestedPaging || (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
3752
3753 /*
3754 * Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW).
3755 */
3756 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3757 uint32_t fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3758 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3759 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
3760 else
3761 Assert((fSetCr0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3762
3763 u32GuestCr0 |= fSetCr0;
3764 u32GuestCr0 &= fZapCr0;
3765 u32GuestCr0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3766
3767 /*
3768 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3769 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3770 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3771 */
3772 uint32_t u32Cr0Mask = X86_CR0_PE
3773 | X86_CR0_NE
3774 | (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
3775 | X86_CR0_PG
3776 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3777 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3778 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3779
3780 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3781 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3782 * and @bugref{6944}. */
3783#if 0
3784 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3785 u32Cr0Mask &= ~X86_CR0_PE;
3786#endif
3787 /*
3788 * Finally, update VMCS fields with the CR0 values and the exception bitmap.
3789 */
3790 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCr0);
3791 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32ShadowCr0);
3792 if (u32Cr0Mask != pVCpu->hm.s.vmx.u32Cr0Mask)
3793 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32Cr0Mask);
3794 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
3795 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3796 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3797 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3798 AssertRCReturn(rc, rc);
3799
3800 /* Update our caches. */
3801 pVCpu->hm.s.vmx.u32Cr0Mask = u32Cr0Mask;
3802 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
3803 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3804
3805 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3806
3807 Log4Func(("u32Cr0Mask=%#RX32 u32ShadowCr0=%#RX32 u32GuestCr0=%#RX32 (fSetCr0=%#RX32 fZapCr0=%#RX32\n", u32Cr0Mask,
3808 u32ShadowCr0, u32GuestCr0, fSetCr0, fZapCr0));
3809 }
3810
3811 return VINF_SUCCESS;
3812}
3813
3814
3815/**
3816 * Exports the guest control registers (CR3, CR4) into the guest-state area
3817 * in the VMCS.
3818 *
3819 * @returns VBox strict status code.
3820 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3821 * without unrestricted guest access and the VMMDev is not presently
3822 * mapped (e.g. EFI32).
3823 *
3824 * @param pVCpu The cross context virtual CPU structure.
3825 *
3826 * @remarks No-long-jump zone!!!
3827 */
3828static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu)
3829{
3830 int rc = VINF_SUCCESS;
3831 PVM pVM = pVCpu->CTX_SUFF(pVM);
3832
3833 /*
3834 * Guest CR2.
3835 * It's always loaded in the assembler code. Nothing to do here.
3836 */
3837
3838 /*
3839 * Guest CR3.
3840 */
3841 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
3842 {
3843 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
3844
3845 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3846 if (pVM->hm.s.fNestedPaging)
3847 {
3848 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3849
3850 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3851 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3852 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3853 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3854
3855 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3856 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3857 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3858
3859 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3860 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3861 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3862 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3863 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3864 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3865 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3866
3867 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3868 AssertRCReturn(rc, rc);
3869
3870 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3871 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3872 || CPUMIsGuestPagingEnabledEx(pCtx))
3873 {
3874 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3875 if (CPUMIsGuestInPAEModeEx(pCtx))
3876 {
3877 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3878 AssertRCReturn(rc, rc);
3879 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3880 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3881 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3882 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3883 AssertRCReturn(rc, rc);
3884 }
3885
3886 /*
3887 * The guest's view of its CR3 is unblemished with Nested Paging when the
3888 * guest is using paging or we have unrestricted guest execution to handle
3889 * the guest when it's not using paging.
3890 */
3891 GCPhysGuestCR3 = pCtx->cr3;
3892 }
3893 else
3894 {
3895 /*
3896 * The guest is not using paging, but the CPU (VT-x) has to. While the guest
3897 * thinks it accesses physical memory directly, we use our identity-mapped
3898 * page table to map guest-linear to guest-physical addresses. EPT takes care
3899 * of translating it to host-physical addresses.
3900 */
3901 RTGCPHYS GCPhys;
3902 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3903
3904 /* We obtain it here every time as the guest could have relocated this PCI region. */
3905 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3906 if (RT_SUCCESS(rc))
3907 { /* likely */ }
3908 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3909 {
3910 Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3911 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3912 }
3913 else
3914 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3915
3916 GCPhysGuestCR3 = GCPhys;
3917 }
3918
3919 Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
3920 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3921 AssertRCReturn(rc, rc);
3922 }
3923 else
3924 {
3925 /* Non-nested paging case, just use the hypervisor's CR3. */
3926 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3927
3928 Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
3929 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3930 AssertRCReturn(rc, rc);
3931 }
3932
3933 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3934 }
3935
3936 /*
3937 * Guest CR4.
3938 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3939 */
3940 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
3941 {
3942 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3943 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
3944 Assert(!RT_HI_U32(pCtx->cr4));
3945
3946 uint32_t u32GuestCr4 = pCtx->cr4;
3947 uint32_t const u32ShadowCr4 = pCtx->cr4;
3948
3949 /*
3950 * Setup VT-x's view of the guest CR4.
3951 *
3952 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3953 * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3954 * redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3955 *
3956 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3957 */
3958 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3959 {
3960 Assert(pVM->hm.s.vmx.pRealModeTSS);
3961 Assert(PDMVmmDevHeapIsEnabled(pVM));
3962 u32GuestCr4 &= ~X86_CR4_VME;
3963 }
3964
3965 if (pVM->hm.s.fNestedPaging)
3966 {
3967 if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3968 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3969 {
3970 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3971 u32GuestCr4 |= X86_CR4_PSE;
3972 /* Our identity mapping is a 32-bit page directory. */
3973 u32GuestCr4 &= ~X86_CR4_PAE;
3974 }
3975 /* else use guest CR4.*/
3976 }
3977 else
3978 {
3979 /*
3980 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3981 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3982 */
3983 switch (pVCpu->hm.s.enmShadowMode)
3984 {
3985 case PGMMODE_REAL: /* Real-mode. */
3986 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3987 case PGMMODE_32_BIT: /* 32-bit paging. */
3988 {
3989 u32GuestCr4 &= ~X86_CR4_PAE;
3990 break;
3991 }
3992
3993 case PGMMODE_PAE: /* PAE paging. */
3994 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3995 {
3996 u32GuestCr4 |= X86_CR4_PAE;
3997 break;
3998 }
3999
4000 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4001 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4002#ifdef VBOX_ENABLE_64_BITS_GUESTS
4003 break;
4004#endif
4005 default:
4006 AssertFailed();
4007 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4008 }
4009 }
4010
4011 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4012 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4013 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4014 u32GuestCr4 |= fSetCr4;
4015 u32GuestCr4 &= fZapCr4;
4016
4017 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them,
4018 that would cause a VM-exit. */
4019 uint32_t u32Cr4Mask = X86_CR4_VME
4020 | X86_CR4_PAE
4021 | X86_CR4_PGE
4022 | X86_CR4_PSE
4023 | X86_CR4_VMXE;
4024 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4025 u32Cr4Mask |= X86_CR4_OSXSAVE;
4026 if (pVM->cpum.ro.GuestFeatures.fPcid)
4027 u32Cr4Mask |= X86_CR4_PCIDE;
4028
4029 /* Write VT-x's view of the guest CR4, the CR4 modify mask and the read-only CR4 shadow
4030 into the VMCS and update our cache. */
4031 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCr4);
4032 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32ShadowCr4);
4033 if (pVCpu->hm.s.vmx.u32Cr4Mask != u32Cr4Mask)
4034 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32Cr4Mask);
4035 AssertRCReturn(rc, rc);
4036 pVCpu->hm.s.vmx.u32Cr4Mask = u32Cr4Mask;
4037
4038 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4039 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
4040
4041 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
4042
4043 Log4Func(("u32GuestCr4=%#RX32 u32ShadowCr4=%#RX32 (fSetCr4=%#RX32 fZapCr4=%#RX32)\n", u32GuestCr4, u32ShadowCr4, fSetCr4,
4044 fZapCr4));
4045 }
4046 return rc;
4047}
4048
4049
4050/**
4051 * Exports the guest debug registers into the guest-state area in the VMCS.
4052 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4053 *
4054 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4055 *
4056 * @returns VBox status code.
4057 * @param pVCpu The cross context virtual CPU structure.
4058 *
4059 * @remarks No-long-jump zone!!!
4060 */
4061static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu)
4062{
4063 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4064
4065#ifdef VBOX_STRICT
4066 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4067 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
4068 {
4069 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4070 Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0);
4071 Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
4072 }
4073#endif
4074
4075 bool fSteppingDB = false;
4076 bool fInterceptMovDRx = false;
4077 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
4078 if (pVCpu->hm.s.fSingleInstruction)
4079 {
4080 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4081 PVM pVM = pVCpu->CTX_SUFF(pVM);
4082 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
4083 {
4084 uProcCtls |= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
4085 Assert(fSteppingDB == false);
4086 }
4087 else
4088 {
4089 pVCpu->cpum.GstCtx.eflags.u32 |= X86_EFL_TF;
4090 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_RFLAGS;
4091 pVCpu->hm.s.fClearTrapFlag = true;
4092 fSteppingDB = true;
4093 }
4094 }
4095
4096 uint32_t u32GuestDr7;
4097 if ( fSteppingDB
4098 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4099 {
4100 /*
4101 * Use the combined guest and host DRx values found in the hypervisor register set
4102 * because the debugger has breakpoints active or someone is single stepping on the
4103 * host side without a monitor trap flag.
4104 *
4105 * Note! DBGF expects a clean DR6 state before executing guest code.
4106 */
4107#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4108 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4109 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4110 {
4111 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4112 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4113 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4114 }
4115 else
4116#endif
4117 if (!CPUMIsHyperDebugStateActive(pVCpu))
4118 {
4119 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4120 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4121 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4122 }
4123
4124 /* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
4125 u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
4126 pVCpu->hm.s.fUsingHyperDR7 = true;
4127 fInterceptMovDRx = true;
4128 }
4129 else
4130 {
4131 /*
4132 * If the guest has enabled debug registers, we need to load them prior to
4133 * executing guest code so they'll trigger at the right time.
4134 */
4135 if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD))
4136 {
4137#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4138 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4139 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4140 {
4141 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4142 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4143 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4144 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4145 }
4146 else
4147#endif
4148 if (!CPUMIsGuestDebugStateActive(pVCpu))
4149 {
4150 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4151 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4152 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4153 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4154 }
4155 Assert(!fInterceptMovDRx);
4156 }
4157 /*
4158 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4159 * must intercept #DB in order to maintain a correct DR6 guest value, and
4160 * because we need to intercept it to prevent nested #DBs from hanging the
4161 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4162 */
4163#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4164 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4165 && !CPUMIsGuestDebugStateActive(pVCpu))
4166#else
4167 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4168#endif
4169 {
4170 fInterceptMovDRx = true;
4171 }
4172
4173 /* Update DR7 with the actual guest value. */
4174 u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
4175 pVCpu->hm.s.fUsingHyperDR7 = false;
4176 }
4177
4178 if (fInterceptMovDRx)
4179 uProcCtls |= VMX_PROC_CTLS_MOV_DR_EXIT;
4180 else
4181 uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
4182
4183 /*
4184 * Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
4185 * monitor-trap flag and update our cache.
4186 */
4187 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
4188 {
4189 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4190 AssertRCReturn(rc2, rc2);
4191 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
4192 }
4193
4194 /*
4195 * Update guest DR7.
4196 */
4197 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
4198 AssertRCReturn(rc, rc);
4199
4200 return VINF_SUCCESS;
4201}
4202
4203
4204#ifdef VBOX_STRICT
4205/**
4206 * Strict function to validate segment registers.
4207 *
4208 * @param pVCpu The cross context virtual CPU structure.
4209 *
4210 * @remarks Will import guest CR0 on strict builds during validation of
4211 * segments.
4212 */
4213static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu)
4214{
4215 /*
4216 * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
4217 *
4218 * The reason we check for attribute value 0 in this function and not just the unusable bit is
4219 * because hmR0VmxExportGuestSegmentReg() only updates the VMCS' copy of the value with the unusable bit
4220 * and doesn't change the guest-context value.
4221 */
4222 PVM pVM = pVCpu->CTX_SUFF(pVM);
4223 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4224 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
4225 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4226 && ( !CPUMIsGuestInRealModeEx(pCtx)
4227 && !CPUMIsGuestInV86ModeEx(pCtx)))
4228 {
4229 /* Protected mode checks */
4230 /* CS */
4231 Assert(pCtx->cs.Attr.n.u1Present);
4232 Assert(!(pCtx->cs.Attr.u & 0xf00));
4233 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4234 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4235 || !(pCtx->cs.Attr.n.u1Granularity));
4236 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4237 || (pCtx->cs.Attr.n.u1Granularity));
4238 /* CS cannot be loaded with NULL in protected mode. */
4239 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4240 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4241 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4242 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4243 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4244 else
4245 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4246 /* SS */
4247 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4248 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4249 if ( !(pCtx->cr0 & X86_CR0_PE)
4250 || pCtx->cs.Attr.n.u4Type == 3)
4251 {
4252 Assert(!pCtx->ss.Attr.n.u2Dpl);
4253 }
4254 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4255 {
4256 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4257 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4258 Assert(pCtx->ss.Attr.n.u1Present);
4259 Assert(!(pCtx->ss.Attr.u & 0xf00));
4260 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4261 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4262 || !(pCtx->ss.Attr.n.u1Granularity));
4263 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4264 || (pCtx->ss.Attr.n.u1Granularity));
4265 }
4266 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmentReg(). */
4267 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4268 {
4269 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4270 Assert(pCtx->ds.Attr.n.u1Present);
4271 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4272 Assert(!(pCtx->ds.Attr.u & 0xf00));
4273 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4274 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4275 || !(pCtx->ds.Attr.n.u1Granularity));
4276 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4277 || (pCtx->ds.Attr.n.u1Granularity));
4278 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4279 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4280 }
4281 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4282 {
4283 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4284 Assert(pCtx->es.Attr.n.u1Present);
4285 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4286 Assert(!(pCtx->es.Attr.u & 0xf00));
4287 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4288 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4289 || !(pCtx->es.Attr.n.u1Granularity));
4290 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4291 || (pCtx->es.Attr.n.u1Granularity));
4292 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4293 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4294 }
4295 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4296 {
4297 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4298 Assert(pCtx->fs.Attr.n.u1Present);
4299 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4300 Assert(!(pCtx->fs.Attr.u & 0xf00));
4301 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4302 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4303 || !(pCtx->fs.Attr.n.u1Granularity));
4304 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4305 || (pCtx->fs.Attr.n.u1Granularity));
4306 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4307 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4308 }
4309 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4310 {
4311 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4312 Assert(pCtx->gs.Attr.n.u1Present);
4313 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4314 Assert(!(pCtx->gs.Attr.u & 0xf00));
4315 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4316 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4317 || !(pCtx->gs.Attr.n.u1Granularity));
4318 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4319 || (pCtx->gs.Attr.n.u1Granularity));
4320 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4321 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4322 }
4323 /* 64-bit capable CPUs. */
4324# if HC_ARCH_BITS == 64
4325 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4326 Assert(!pCtx->ss.Attr.u || !RT_HI_U32(pCtx->ss.u64Base));
4327 Assert(!pCtx->ds.Attr.u || !RT_HI_U32(pCtx->ds.u64Base));
4328 Assert(!pCtx->es.Attr.u || !RT_HI_U32(pCtx->es.u64Base));
4329# endif
4330 }
4331 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4332 || ( CPUMIsGuestInRealModeEx(pCtx)
4333 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4334 {
4335 /* Real and v86 mode checks. */
4336 /* hmR0VmxExportGuestSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4337 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4338 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4339 {
4340 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4341 }
4342 else
4343 {
4344 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4345 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4346 }
4347
4348 /* CS */
4349 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4350 Assert(pCtx->cs.u32Limit == 0xffff);
4351 Assert(u32CSAttr == 0xf3);
4352 /* SS */
4353 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4354 Assert(pCtx->ss.u32Limit == 0xffff);
4355 Assert(u32SSAttr == 0xf3);
4356 /* DS */
4357 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4358 Assert(pCtx->ds.u32Limit == 0xffff);
4359 Assert(u32DSAttr == 0xf3);
4360 /* ES */
4361 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4362 Assert(pCtx->es.u32Limit == 0xffff);
4363 Assert(u32ESAttr == 0xf3);
4364 /* FS */
4365 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4366 Assert(pCtx->fs.u32Limit == 0xffff);
4367 Assert(u32FSAttr == 0xf3);
4368 /* GS */
4369 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4370 Assert(pCtx->gs.u32Limit == 0xffff);
4371 Assert(u32GSAttr == 0xf3);
4372 /* 64-bit capable CPUs. */
4373# if HC_ARCH_BITS == 64
4374 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4375 Assert(!u32SSAttr || !RT_HI_U32(pCtx->ss.u64Base));
4376 Assert(!u32DSAttr || !RT_HI_U32(pCtx->ds.u64Base));
4377 Assert(!u32ESAttr || !RT_HI_U32(pCtx->es.u64Base));
4378# endif
4379 }
4380}
4381#endif /* VBOX_STRICT */
4382
4383
4384/**
4385 * Exports a guest segment register into the guest-state area in the VMCS.
4386 *
4387 * @returns VBox status code.
4388 * @param pVCpu The cross context virtual CPU structure.
4389 * @param idxSel Index of the selector in the VMCS.
4390 * @param idxLimit Index of the segment limit in the VMCS.
4391 * @param idxBase Index of the segment base in the VMCS.
4392 * @param idxAccess Index of the access rights of the segment in the VMCS.
4393 * @param pSelReg Pointer to the segment selector.
4394 *
4395 * @remarks No-long-jump zone!!!
4396 */
4397static int hmR0VmxExportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
4398 PCCPUMSELREG pSelReg)
4399{
4400 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4401 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4402 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4403 AssertRCReturn(rc, rc);
4404
4405 uint32_t u32Access = pSelReg->Attr.u;
4406 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4407 {
4408 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4409 u32Access = 0xf3;
4410 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4411 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4412 }
4413 else
4414 {
4415 /*
4416 * The way to differentiate between whether this is really a null selector or was just
4417 * a selector loaded with 0 in real-mode is using the segment attributes. A selector
4418 * loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4419 * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4420 * NULL selectors loaded in protected-mode have their attribute as 0.
4421 */
4422 if (!u32Access)
4423 u32Access = X86DESCATTR_UNUSABLE;
4424 }
4425
4426 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4427 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4428 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4429
4430 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4431 AssertRCReturn(rc, rc);
4432 return rc;
4433}
4434
4435
4436/**
4437 * Exports the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4438 * into the guest-state area in the VMCS.
4439 *
4440 * @returns VBox status code.
4441 * @param pVCpu The cross context virtual CPU structure.
4442 *
4443 * @remarks Will import guest CR0 on strict builds during validation of
4444 * segments.
4445 * @remarks No-long-jump zone!!!
4446 */
4447static int hmR0VmxExportGuestSegmentRegs(PVMCPU pVCpu)
4448{
4449 int rc = VERR_INTERNAL_ERROR_5;
4450 PVM pVM = pVCpu->CTX_SUFF(pVM);
4451 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4452
4453 /*
4454 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4455 */
4456 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4457 {
4458#ifdef VBOX_WITH_REM
4459 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4460 {
4461 Assert(pVM->hm.s.vmx.pRealModeTSS);
4462 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4463 if ( pVCpu->hm.s.vmx.fWasInRealMode
4464 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4465 {
4466 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4467 in real-mode (e.g. OpenBSD 4.0) */
4468 REMFlushTBs(pVM);
4469 Log4Func(("Switch to protected mode detected!\n"));
4470 pVCpu->hm.s.vmx.fWasInRealMode = false;
4471 }
4472 }
4473#endif
4474 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
4475 {
4476 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
4477 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4478 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pCtx->cs.Attr.u;
4479 rc = HMVMX_EXPORT_SREG(CS, &pCtx->cs);
4480 AssertRCReturn(rc, rc);
4481 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
4482 }
4483
4484 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
4485 {
4486 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
4487 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4488 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pCtx->ss.Attr.u;
4489 rc = HMVMX_EXPORT_SREG(SS, &pCtx->ss);
4490 AssertRCReturn(rc, rc);
4491 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
4492 }
4493
4494 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
4495 {
4496 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
4497 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4498 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pCtx->ds.Attr.u;
4499 rc = HMVMX_EXPORT_SREG(DS, &pCtx->ds);
4500 AssertRCReturn(rc, rc);
4501 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
4502 }
4503
4504 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
4505 {
4506 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
4507 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4508 pVCpu->hm.s.vmx.RealMode.AttrES.u = pCtx->es.Attr.u;
4509 rc = HMVMX_EXPORT_SREG(ES, &pCtx->es);
4510 AssertRCReturn(rc, rc);
4511 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
4512 }
4513
4514 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
4515 {
4516 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
4517 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4518 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pCtx->fs.Attr.u;
4519 rc = HMVMX_EXPORT_SREG(FS, &pCtx->fs);
4520 AssertRCReturn(rc, rc);
4521 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
4522 }
4523
4524 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
4525 {
4526 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
4527 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4528 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pCtx->gs.Attr.u;
4529 rc = HMVMX_EXPORT_SREG(GS, &pCtx->gs);
4530 AssertRCReturn(rc, rc);
4531 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
4532 }
4533
4534#ifdef VBOX_STRICT
4535 hmR0VmxValidateSegmentRegs(pVCpu);
4536#endif
4537
4538 Log4Func(("CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pCtx->cs.Sel, pCtx->cs.u64Base,
4539 pCtx->cs.u32Limit, pCtx->cs.Attr.u));
4540 }
4541
4542 /*
4543 * Guest TR.
4544 */
4545 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
4546 {
4547 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
4548
4549 /*
4550 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
4551 * achieved using the interrupt redirection bitmap (all bits cleared to let the guest
4552 * handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4553 */
4554 uint16_t u16Sel = 0;
4555 uint32_t u32Limit = 0;
4556 uint64_t u64Base = 0;
4557 uint32_t u32AccessRights = 0;
4558
4559 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4560 {
4561 u16Sel = pCtx->tr.Sel;
4562 u32Limit = pCtx->tr.u32Limit;
4563 u64Base = pCtx->tr.u64Base;
4564 u32AccessRights = pCtx->tr.Attr.u;
4565 }
4566 else
4567 {
4568 Assert(pVM->hm.s.vmx.pRealModeTSS);
4569 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4570
4571 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4572 RTGCPHYS GCPhys;
4573 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4574 AssertRCReturn(rc, rc);
4575
4576 X86DESCATTR DescAttr;
4577 DescAttr.u = 0;
4578 DescAttr.n.u1Present = 1;
4579 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4580
4581 u16Sel = 0;
4582 u32Limit = HM_VTX_TSS_SIZE;
4583 u64Base = GCPhys; /* in real-mode phys = virt. */
4584 u32AccessRights = DescAttr.u;
4585 }
4586
4587 /* Validate. */
4588 Assert(!(u16Sel & RT_BIT(2)));
4589 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4590 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4591 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4592 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4593 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4594 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4595 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4596 Assert( (u32Limit & 0xfff) == 0xfff
4597 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4598 Assert( !(pCtx->tr.u32Limit & 0xfff00000)
4599 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4600
4601 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4602 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4603 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4604 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4605 AssertRCReturn(rc, rc);
4606
4607 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
4608 Log4Func(("TR base=%#RX64\n", pCtx->tr.u64Base));
4609 }
4610
4611 /*
4612 * Guest GDTR.
4613 */
4614 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4615 {
4616 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
4617
4618 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
4619 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
4620 AssertRCReturn(rc, rc);
4621
4622 /* Validate. */
4623 Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4624
4625 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4626 Log4Func(("GDTR base=%#RX64\n", pCtx->gdtr.pGdt));
4627 }
4628
4629 /*
4630 * Guest LDTR.
4631 */
4632 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4633 {
4634 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
4635
4636 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4637 uint32_t u32Access = 0;
4638 if (!pCtx->ldtr.Attr.u)
4639 u32Access = X86DESCATTR_UNUSABLE;
4640 else
4641 u32Access = pCtx->ldtr.Attr.u;
4642
4643 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
4644 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
4645 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4646 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
4647 AssertRCReturn(rc, rc);
4648
4649 /* Validate. */
4650 if (!(u32Access & X86DESCATTR_UNUSABLE))
4651 {
4652 Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4653 Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4654 Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4655 Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4656 Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4657 Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4658 Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
4659 || !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4660 Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
4661 || pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4662 }
4663
4664 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4665 Log4Func(("LDTR base=%#RX64\n", pCtx->ldtr.u64Base));
4666 }
4667
4668 /*
4669 * Guest IDTR.
4670 */
4671 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4672 {
4673 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
4674
4675 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
4676 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
4677 AssertRCReturn(rc, rc);
4678
4679 /* Validate. */
4680 Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4681
4682 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4683 Log4Func(("IDTR base=%#RX64\n", pCtx->idtr.pIdt));
4684 }
4685
4686 return VINF_SUCCESS;
4687}
4688
4689
4690/**
4691 * Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4692 * areas.
4693 *
4694 * These MSRs will automatically be loaded to the host CPU on every successful
4695 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4696 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4697 * -not- updated here for performance reasons. See hmR0VmxExportHostMsrs().
4698 *
4699 * Also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4700 *
4701 * @returns VBox status code.
4702 * @param pVCpu The cross context virtual CPU structure.
4703 *
4704 * @remarks No-long-jump zone!!!
4705 */
4706static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu)
4707{
4708 AssertPtr(pVCpu);
4709 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4710
4711 /*
4712 * MSRs that we use the auto-load/store MSR area in the VMCS.
4713 * For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
4714 */
4715 PVM pVM = pVCpu->CTX_SUFF(pVM);
4716 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4717 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4718 {
4719 if (pVM->hm.s.fAllow64BitGuests)
4720 {
4721#if HC_ARCH_BITS == 32
4722 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS | CPUMCTX_EXTRN_KERNEL_GS_BASE);
4723
4724 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pCtx->msrLSTAR, false, NULL);
4725 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pCtx->msrSTAR, false, NULL);
4726 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pCtx->msrSFMASK, false, NULL);
4727 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, false, NULL);
4728 AssertRCReturn(rc, rc);
4729# ifdef LOG_ENABLED
4730 PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4731 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4732 Log4Func(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4733# endif
4734#endif
4735 }
4736 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4737 }
4738
4739 /*
4740 * Guest Sysenter MSRs.
4741 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4742 * VM-exits on WRMSRs for these MSRs.
4743 */
4744 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4745 {
4746 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
4747
4748 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4749 {
4750 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
4751 AssertRCReturn(rc, rc);
4752 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4753 }
4754
4755 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4756 {
4757 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
4758 AssertRCReturn(rc, rc);
4759 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4760 }
4761
4762 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4763 {
4764 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
4765 AssertRCReturn(rc, rc);
4766 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4767 }
4768 }
4769
4770 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4771 {
4772 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
4773
4774 if (hmR0VmxShouldSwapEferMsr(pVCpu))
4775 {
4776 /*
4777 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4778 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4779 */
4780 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4781 {
4782 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
4783 AssertRCReturn(rc,rc);
4784 Log4Func(("EFER=%#RX64\n", pCtx->msrEFER));
4785 }
4786 else
4787 {
4788 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pCtx->msrEFER, false /* fUpdateHostMsr */,
4789 NULL /* pfAddedAndUpdated */);
4790 AssertRCReturn(rc, rc);
4791
4792 /* We need to intercept reads too, see @bugref{7386#c16}. */
4793 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
4794 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4795 Log4Func(("MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pCtx->msrEFER,
4796 pVCpu->hm.s.vmx.cMsrs));
4797 }
4798 }
4799 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4800 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4801 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4802 }
4803
4804 return VINF_SUCCESS;
4805}
4806
4807
4808#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4809/**
4810 * Check if guest state allows safe use of 32-bit switcher again.
4811 *
4812 * Segment bases and protected mode structures must be 32-bit addressable
4813 * because the 32-bit switcher will ignore high dword when writing these VMCS
4814 * fields. See @bugref{8432} for details.
4815 *
4816 * @returns true if safe, false if must continue to use the 64-bit switcher.
4817 * @param pCtx Pointer to the guest-CPU context.
4818 *
4819 * @remarks No-long-jump zone!!!
4820 */
4821static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
4822{
4823 if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4824 if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4825 if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4826 if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4827 if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4828 if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4829 if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4830 if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4831 if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4832 if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4833
4834 /* All good, bases are 32-bit. */
4835 return true;
4836}
4837#endif
4838
4839
4840/**
4841 * Selects up the appropriate function to run guest code.
4842 *
4843 * @returns VBox status code.
4844 * @param pVCpu The cross context virtual CPU structure.
4845 *
4846 * @remarks No-long-jump zone!!!
4847 */
4848static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu)
4849{
4850 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4851 if (CPUMIsGuestInLongModeEx(pCtx))
4852 {
4853#ifndef VBOX_ENABLE_64_BITS_GUESTS
4854 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4855#endif
4856 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4857#if HC_ARCH_BITS == 32
4858 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4859 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4860 {
4861#ifdef VBOX_STRICT
4862 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4863 {
4864 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4865 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4866 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4867 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4868 | HM_CHANGED_VMX_ENTRY_CTLS
4869 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4870 }
4871#endif
4872 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4873
4874 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4875 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4876 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4877 Log4Func(("Selected 64-bit switcher\n"));
4878 }
4879#else
4880 /* 64-bit host. */
4881 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4882#endif
4883 }
4884 else
4885 {
4886 /* Guest is not in long mode, use the 32-bit handler. */
4887#if HC_ARCH_BITS == 32
4888 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4889 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4890 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4891 {
4892# ifdef VBOX_STRICT
4893 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4894 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4895 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4896 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4897 | HM_CHANGED_VMX_ENTRY_CTLS
4898 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4899# endif
4900 }
4901# ifdef VBOX_ENABLE_64_BITS_GUESTS
4902 /*
4903 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
4904 * design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
4905 * switcher flag because now we know the guest is in a sane state where it's safe
4906 * to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4907 * the much faster 32-bit switcher again.
4908 */
4909 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4910 {
4911 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4912 Log4Func(("Selected 32-bit switcher\n"));
4913 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4914 }
4915 else
4916 {
4917 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4918 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4919 || hmR0VmxIs32BitSwitcherSafe(pCtx))
4920 {
4921 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4922 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4923 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
4924 | HM_CHANGED_VMX_ENTRY_CTLS
4925 | HM_CHANGED_VMX_EXIT_CTLS
4926 | HM_CHANGED_HOST_CONTEXT);
4927 Log4Func(("Selected 32-bit switcher (safe)\n"));
4928 }
4929 }
4930# else
4931 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4932# endif
4933#else
4934 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4935#endif
4936 }
4937 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4938 return VINF_SUCCESS;
4939}
4940
4941
4942/**
4943 * Wrapper for running the guest code in VT-x.
4944 *
4945 * @returns VBox status code, no informational status codes.
4946 * @param pVCpu The cross context virtual CPU structure.
4947 *
4948 * @remarks No-long-jump zone!!!
4949 */
4950DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu)
4951{
4952 /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4953 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4954 pCtx->fExtrn |= HMVMX_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
4955
4956 /*
4957 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
4958 * floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
4959 * callee-saved and thus the need for this XMM wrapper.
4960 *
4961 * See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
4962 */
4963 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4964 /** @todo Add stats for resume vs launch. */
4965 PVM pVM = pVCpu->CTX_SUFF(pVM);
4966#ifdef VBOX_WITH_KERNEL_USING_XMM
4967 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4968#else
4969 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4970#endif
4971 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4972 return rc;
4973}
4974
4975
4976/**
4977 * Reports world-switch error and dumps some useful debug info.
4978 *
4979 * @param pVCpu The cross context virtual CPU structure.
4980 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4981 * @param pVmxTransient Pointer to the VMX transient structure (only
4982 * exitReason updated).
4983 */
4984static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
4985{
4986 Assert(pVCpu);
4987 Assert(pVmxTransient);
4988 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
4989
4990 Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
4991 switch (rcVMRun)
4992 {
4993 case VERR_VMX_INVALID_VMXON_PTR:
4994 AssertFailed();
4995 break;
4996 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4997 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4998 {
4999 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5000 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5001 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5002 AssertRC(rc);
5003
5004 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5005 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5006 Cannot do it here as we may have been long preempted. */
5007
5008#ifdef VBOX_STRICT
5009 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5010 pVmxTransient->uExitReason));
5011 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5012 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5013 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5014 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5015 else
5016 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5017 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5018 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5019
5020 /* VMX control bits. */
5021 uint32_t u32Val;
5022 uint64_t u64Val;
5023 RTHCUINTREG uHCReg;
5024 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5025 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5026 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5027 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5028 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
5029 {
5030 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5031 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5032 }
5033 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5034 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5035 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5036 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5037 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5038 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5039 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5040 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5041 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5042 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5043 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5044 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5045 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5046 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5047 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5048 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5049 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5050 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5051 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5052 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5053 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5054 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5055 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5056 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5057 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5058 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5059 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5060 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5061 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5062 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5063 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5064 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5065 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5066 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5067 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5068 {
5069 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5070 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5071 }
5072
5073 /* Guest bits. */
5074 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5075 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
5076 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5077 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
5078 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5079 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
5080 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
5081 {
5082 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5083 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5084 }
5085
5086 /* Host bits. */
5087 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5088 Log4(("Host CR0 %#RHr\n", uHCReg));
5089 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5090 Log4(("Host CR3 %#RHr\n", uHCReg));
5091 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5092 Log4(("Host CR4 %#RHr\n", uHCReg));
5093
5094 RTGDTR HostGdtr;
5095 PCX86DESCHC pDesc;
5096 ASMGetGDTR(&HostGdtr);
5097 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5098 Log4(("Host CS %#08x\n", u32Val));
5099 if (u32Val < HostGdtr.cbGdt)
5100 {
5101 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5102 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5103 }
5104
5105 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5106 Log4(("Host DS %#08x\n", u32Val));
5107 if (u32Val < HostGdtr.cbGdt)
5108 {
5109 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5110 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5111 }
5112
5113 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5114 Log4(("Host ES %#08x\n", u32Val));
5115 if (u32Val < HostGdtr.cbGdt)
5116 {
5117 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5118 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5119 }
5120
5121 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5122 Log4(("Host FS %#08x\n", u32Val));
5123 if (u32Val < HostGdtr.cbGdt)
5124 {
5125 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5126 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5127 }
5128
5129 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5130 Log4(("Host GS %#08x\n", u32Val));
5131 if (u32Val < HostGdtr.cbGdt)
5132 {
5133 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5134 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5135 }
5136
5137 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5138 Log4(("Host SS %#08x\n", u32Val));
5139 if (u32Val < HostGdtr.cbGdt)
5140 {
5141 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5142 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5143 }
5144
5145 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5146 Log4(("Host TR %#08x\n", u32Val));
5147 if (u32Val < HostGdtr.cbGdt)
5148 {
5149 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5150 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5151 }
5152
5153 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5154 Log4(("Host TR Base %#RHv\n", uHCReg));
5155 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5156 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5157 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5158 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5159 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5160 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5161 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5162 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5163 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5164 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5165 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5166 Log4(("Host RSP %#RHv\n", uHCReg));
5167 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5168 Log4(("Host RIP %#RHv\n", uHCReg));
5169# if HC_ARCH_BITS == 64
5170 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5171 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5172 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5173 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5174 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5175 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5176# endif
5177#endif /* VBOX_STRICT */
5178 break;
5179 }
5180
5181 default:
5182 /* Impossible */
5183 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5184 break;
5185 }
5186}
5187
5188
5189#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5190#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5191# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5192#endif
5193#ifdef VBOX_STRICT
5194static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5195{
5196 switch (idxField)
5197 {
5198 case VMX_VMCS_GUEST_RIP:
5199 case VMX_VMCS_GUEST_RSP:
5200 case VMX_VMCS_GUEST_SYSENTER_EIP:
5201 case VMX_VMCS_GUEST_SYSENTER_ESP:
5202 case VMX_VMCS_GUEST_GDTR_BASE:
5203 case VMX_VMCS_GUEST_IDTR_BASE:
5204 case VMX_VMCS_GUEST_CS_BASE:
5205 case VMX_VMCS_GUEST_DS_BASE:
5206 case VMX_VMCS_GUEST_ES_BASE:
5207 case VMX_VMCS_GUEST_FS_BASE:
5208 case VMX_VMCS_GUEST_GS_BASE:
5209 case VMX_VMCS_GUEST_SS_BASE:
5210 case VMX_VMCS_GUEST_LDTR_BASE:
5211 case VMX_VMCS_GUEST_TR_BASE:
5212 case VMX_VMCS_GUEST_CR3:
5213 return true;
5214 }
5215 return false;
5216}
5217
5218static bool hmR0VmxIsValidReadField(uint32_t idxField)
5219{
5220 switch (idxField)
5221 {
5222 /* Read-only fields. */
5223 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5224 return true;
5225 }
5226 /* Remaining readable fields should also be writable. */
5227 return hmR0VmxIsValidWriteField(idxField);
5228}
5229#endif /* VBOX_STRICT */
5230
5231
5232/**
5233 * Executes the specified handler in 64-bit mode.
5234 *
5235 * @returns VBox status code (no informational status codes).
5236 * @param pVCpu The cross context virtual CPU structure.
5237 * @param enmOp The operation to perform.
5238 * @param cParams Number of parameters.
5239 * @param paParam Array of 32-bit parameters.
5240 */
5241VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
5242{
5243 PVM pVM = pVCpu->CTX_SUFF(pVM);
5244 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5245 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5246 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5247 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5248
5249#ifdef VBOX_STRICT
5250 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5251 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5252
5253 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5254 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5255#endif
5256
5257 /* Disable interrupts. */
5258 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5259
5260#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5261 RTCPUID idHostCpu = RTMpCpuId();
5262 CPUMR0SetLApic(pVCpu, idHostCpu);
5263#endif
5264
5265 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5266 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5267
5268 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5269 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5270 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5271
5272 /* Leave VMX Root Mode. */
5273 VMXDisable();
5274
5275 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5276
5277 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5278 CPUMSetHyperEIP(pVCpu, enmOp);
5279 for (int i = (int)cParams - 1; i >= 0; i--)
5280 CPUMPushHyper(pVCpu, paParam[i]);
5281
5282 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5283
5284 /* Call the switcher. */
5285 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
5286 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5287
5288 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5289 /* Make sure the VMX instructions don't cause #UD faults. */
5290 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5291
5292 /* Re-enter VMX Root Mode */
5293 int rc2 = VMXEnable(HCPhysCpuPage);
5294 if (RT_FAILURE(rc2))
5295 {
5296 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5297 ASMSetFlags(fOldEFlags);
5298 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5299 return rc2;
5300 }
5301
5302 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5303 AssertRC(rc2);
5304 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5305 Assert(!(ASMGetFlags() & X86_EFL_IF));
5306 ASMSetFlags(fOldEFlags);
5307 return rc;
5308}
5309
5310
5311/**
5312 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5313 * supporting 64-bit guests.
5314 *
5315 * @returns VBox status code.
5316 * @param fResume Whether to VMLAUNCH or VMRESUME.
5317 * @param pCtx Pointer to the guest-CPU context.
5318 * @param pCache Pointer to the VMCS cache.
5319 * @param pVM The cross context VM structure.
5320 * @param pVCpu The cross context virtual CPU structure.
5321 */
5322DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5323{
5324 NOREF(fResume);
5325
5326 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5327 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5328
5329#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5330 pCache->uPos = 1;
5331 pCache->interPD = PGMGetInterPaeCR3(pVM);
5332 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5333#endif
5334
5335#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5336 pCache->TestIn.HCPhysCpuPage = 0;
5337 pCache->TestIn.HCPhysVmcs = 0;
5338 pCache->TestIn.pCache = 0;
5339 pCache->TestOut.HCPhysVmcs = 0;
5340 pCache->TestOut.pCache = 0;
5341 pCache->TestOut.pCtx = 0;
5342 pCache->TestOut.eflags = 0;
5343#else
5344 NOREF(pCache);
5345#endif
5346
5347 uint32_t aParam[10];
5348 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5349 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5350 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5351 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5352 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5353 aParam[5] = 0;
5354 aParam[6] = VM_RC_ADDR(pVM, pVM);
5355 aParam[7] = 0;
5356 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5357 aParam[9] = 0;
5358
5359#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5360 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5361 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5362#endif
5363 int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5364
5365#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5366 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5367 Assert(pCtx->dr[4] == 10);
5368 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5369#endif
5370
5371#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5372 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5373 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5374 pVCpu->hm.s.vmx.HCPhysVmcs));
5375 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5376 pCache->TestOut.HCPhysVmcs));
5377 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5378 pCache->TestOut.pCache));
5379 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5380 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5381 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5382 pCache->TestOut.pCtx));
5383 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5384#endif
5385 NOREF(pCtx);
5386 return rc;
5387}
5388
5389
5390/**
5391 * Initialize the VMCS-Read cache.
5392 *
5393 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5394 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5395 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5396 * (those that have a 32-bit FULL & HIGH part).
5397 *
5398 * @returns VBox status code.
5399 * @param pVCpu The cross context virtual CPU structure.
5400 */
5401static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
5402{
5403#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5404 do { \
5405 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5406 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5407 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5408 ++cReadFields; \
5409 } while (0)
5410
5411 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5412 uint32_t cReadFields = 0;
5413
5414 /*
5415 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5416 * and serve to indicate exceptions to the rules.
5417 */
5418
5419 /* Guest-natural selector base fields. */
5420#if 0
5421 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5422 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5423 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5424#endif
5425 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5426 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5427 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5428 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5429 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5430 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5431 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5432 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5433 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5434 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5435 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5436 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5437#if 0
5438 /* Unused natural width guest-state fields. */
5439 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5440 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5441#endif
5442 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5443 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5444
5445 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
5446 these 64-bit fields (using "FULL" and "HIGH" fields). */
5447#if 0
5448 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5449 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5450 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5451 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5452 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5453 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5454 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5455 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5456 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5457#endif
5458
5459 /* Natural width guest-state fields. */
5460 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5461#if 0
5462 /* Currently unused field. */
5463 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5464#endif
5465
5466 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5467 {
5468 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5469 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5470 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5471 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5472 }
5473 else
5474 {
5475 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5476 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5477 }
5478
5479#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5480 return VINF_SUCCESS;
5481}
5482
5483
5484/**
5485 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5486 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5487 * darwin, running 64-bit guests).
5488 *
5489 * @returns VBox status code.
5490 * @param pVCpu The cross context virtual CPU structure.
5491 * @param idxField The VMCS field encoding.
5492 * @param u64Val 16, 32 or 64-bit value.
5493 */
5494VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5495{
5496 int rc;
5497 switch (idxField)
5498 {
5499 /*
5500 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5501 */
5502 /* 64-bit Control fields. */
5503 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5504 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5505 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5506 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5507 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5508 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5509 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5510 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5511 case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
5512 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5513 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5514 case VMX_VMCS64_CTRL_EPTP_FULL:
5515 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5516 /* 64-bit Guest-state fields. */
5517 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5518 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5519 case VMX_VMCS64_GUEST_PAT_FULL:
5520 case VMX_VMCS64_GUEST_EFER_FULL:
5521 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5522 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5523 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5524 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5525 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5526 /* 64-bit Host-state fields. */
5527 case VMX_VMCS64_HOST_PAT_FULL:
5528 case VMX_VMCS64_HOST_EFER_FULL:
5529 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5530 {
5531 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5532 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5533 break;
5534 }
5535
5536 /*
5537 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5538 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5539 */
5540 /* Natural-width Guest-state fields. */
5541 case VMX_VMCS_GUEST_CR3:
5542 case VMX_VMCS_GUEST_ES_BASE:
5543 case VMX_VMCS_GUEST_CS_BASE:
5544 case VMX_VMCS_GUEST_SS_BASE:
5545 case VMX_VMCS_GUEST_DS_BASE:
5546 case VMX_VMCS_GUEST_FS_BASE:
5547 case VMX_VMCS_GUEST_GS_BASE:
5548 case VMX_VMCS_GUEST_LDTR_BASE:
5549 case VMX_VMCS_GUEST_TR_BASE:
5550 case VMX_VMCS_GUEST_GDTR_BASE:
5551 case VMX_VMCS_GUEST_IDTR_BASE:
5552 case VMX_VMCS_GUEST_RSP:
5553 case VMX_VMCS_GUEST_RIP:
5554 case VMX_VMCS_GUEST_SYSENTER_ESP:
5555 case VMX_VMCS_GUEST_SYSENTER_EIP:
5556 {
5557 if (!(RT_HI_U32(u64Val)))
5558 {
5559 /* If this field is 64-bit, VT-x will zero out the top bits. */
5560 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5561 }
5562 else
5563 {
5564 /* Assert that only the 32->64 switcher case should ever come here. */
5565 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5566 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5567 }
5568 break;
5569 }
5570
5571 default:
5572 {
5573 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5574 rc = VERR_INVALID_PARAMETER;
5575 break;
5576 }
5577 }
5578 AssertRCReturn(rc, rc);
5579 return rc;
5580}
5581
5582
5583/**
5584 * Queue up a VMWRITE by using the VMCS write cache.
5585 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5586 *
5587 * @param pVCpu The cross context virtual CPU structure.
5588 * @param idxField The VMCS field encoding.
5589 * @param u64Val 16, 32 or 64-bit value.
5590 */
5591VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5592{
5593 AssertPtr(pVCpu);
5594 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5595
5596 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5597 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5598
5599 /* Make sure there are no duplicates. */
5600 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5601 {
5602 if (pCache->Write.aField[i] == idxField)
5603 {
5604 pCache->Write.aFieldVal[i] = u64Val;
5605 return VINF_SUCCESS;
5606 }
5607 }
5608
5609 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5610 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5611 pCache->Write.cValidEntries++;
5612 return VINF_SUCCESS;
5613}
5614#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5615
5616
5617/**
5618 * Sets up the usage of TSC-offsetting and updates the VMCS.
5619 *
5620 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5621 * VMX preemption timer.
5622 *
5623 * @returns VBox status code.
5624 * @param pVCpu The cross context virtual CPU structure.
5625 *
5626 * @remarks No-long-jump zone!!!
5627 */
5628static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5629{
5630 bool fOffsettedTsc;
5631 bool fParavirtTsc;
5632 PVM pVM = pVCpu->CTX_SUFF(pVM);
5633 uint64_t uTscOffset;
5634 if (pVM->hm.s.vmx.fUsePreemptTimer)
5635 {
5636 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
5637
5638 /* Make sure the returned values have sane upper and lower boundaries. */
5639 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5640 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5641 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5642 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5643
5644 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5645 int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
5646 AssertRC(rc);
5647 }
5648 else
5649 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
5650
5651 if (fParavirtTsc)
5652 {
5653 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5654 information before every VM-entry, hence disable it for performance sake. */
5655#if 0
5656 int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5657 AssertRC(rc);
5658#endif
5659 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5660 }
5661
5662 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
5663 if ( fOffsettedTsc
5664 && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5665 {
5666 if (pVCpu->hm.s.vmx.u64TscOffset != uTscOffset)
5667 {
5668 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
5669 AssertRC(rc);
5670 pVCpu->hm.s.vmx.u64TscOffset = uTscOffset;
5671 }
5672
5673 if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
5674 {
5675 uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
5676 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5677 AssertRC(rc);
5678 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5679 }
5680 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5681 }
5682 else
5683 {
5684 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5685 if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
5686 {
5687 uProcCtls |= VMX_PROC_CTLS_RDTSC_EXIT;
5688 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5689 AssertRC(rc);
5690 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5691 }
5692 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5693 }
5694}
5695
5696
5697/**
5698 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5699 * VM-exit interruption info type.
5700 *
5701 * @returns The IEM exception flags.
5702 * @param uVector The event vector.
5703 * @param uVmxVectorType The VMX event type.
5704 *
5705 * @remarks This function currently only constructs flags required for
5706 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5707 * and CR2 aspects of an exception are not included).
5708 */
5709static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5710{
5711 uint32_t fIemXcptFlags;
5712 switch (uVmxVectorType)
5713 {
5714 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5715 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5716 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5717 break;
5718
5719 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5720 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5721 break;
5722
5723 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5724 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5725 break;
5726
5727 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5728 {
5729 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5730 if (uVector == X86_XCPT_BP)
5731 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5732 else if (uVector == X86_XCPT_OF)
5733 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5734 else
5735 {
5736 fIemXcptFlags = 0;
5737 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5738 }
5739 break;
5740 }
5741
5742 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5743 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5744 break;
5745
5746 default:
5747 fIemXcptFlags = 0;
5748 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5749 break;
5750 }
5751 return fIemXcptFlags;
5752}
5753
5754
5755/**
5756 * Sets an event as a pending event to be injected into the guest.
5757 *
5758 * @param pVCpu The cross context virtual CPU structure.
5759 * @param u32IntInfo The VM-entry interruption-information field.
5760 * @param cbInstr The VM-entry instruction length in bytes (for software
5761 * interrupts, exceptions and privileged software
5762 * exceptions).
5763 * @param u32ErrCode The VM-entry exception error code.
5764 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5765 * page-fault.
5766 *
5767 * @remarks Statistics counter assumes this is a guest event being injected or
5768 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5769 * always incremented.
5770 */
5771DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5772 RTGCUINTPTR GCPtrFaultAddress)
5773{
5774 Assert(!pVCpu->hm.s.Event.fPending);
5775 pVCpu->hm.s.Event.fPending = true;
5776 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5777 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5778 pVCpu->hm.s.Event.cbInstr = cbInstr;
5779 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5780}
5781
5782
5783/**
5784 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5785 *
5786 * @param pVCpu The cross context virtual CPU structure.
5787 */
5788DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
5789{
5790 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INT_INFO_VALID;
5791 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
5792 u32IntInfo |= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
5793 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5794}
5795
5796
5797/**
5798 * Handle a condition that occurred while delivering an event through the guest
5799 * IDT.
5800 *
5801 * @returns Strict VBox status code (i.e. informational status codes too).
5802 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5803 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5804 * to continue execution of the guest which will delivery the \#DF.
5805 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5806 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5807 *
5808 * @param pVCpu The cross context virtual CPU structure.
5809 * @param pVmxTransient Pointer to the VMX transient structure.
5810 *
5811 * @remarks No-long-jump zone!!!
5812 */
5813static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5814{
5815 uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5816
5817 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5818 rc2 |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5819 AssertRCReturn(rc2, rc2);
5820
5821 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5822 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5823 {
5824 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5825 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5826
5827 /*
5828 * If the event was a software interrupt (generated with INT n) or a software exception
5829 * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
5830 * can handle the VM-exit and continue guest execution which will re-execute the
5831 * instruction rather than re-injecting the exception, as that can cause premature
5832 * trips to ring-3 before injection and involve TRPM which currently has no way of
5833 * storing that these exceptions were caused by these instructions (ICEBP's #DB poses
5834 * the problem).
5835 */
5836 IEMXCPTRAISE enmRaise;
5837 IEMXCPTRAISEINFO fRaiseInfo;
5838 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5839 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5840 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5841 {
5842 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5843 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5844 }
5845 else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5846 {
5847 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5848 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5849 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5850 /** @todo Make AssertMsgReturn as just AssertMsg later. */
5851 AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
5852 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5853 uExitVectorType), VERR_VMX_IPE_5);
5854
5855 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5856
5857 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5858 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
5859 {
5860 pVmxTransient->fVectoringPF = true;
5861 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5862 }
5863 }
5864 else
5865 {
5866 /*
5867 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5868 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5869 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
5870 */
5871 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5872 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5873 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5874 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5875 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5876 }
5877
5878 /*
5879 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5880 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5881 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5882 * subsequent VM-entry would fail.
5883 *
5884 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5885 */
5886 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
5887 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5888 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
5889 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
5890 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
5891 {
5892 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5893 }
5894
5895 switch (enmRaise)
5896 {
5897 case IEMXCPTRAISE_CURRENT_XCPT:
5898 {
5899 Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
5900 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
5901 Assert(rcStrict == VINF_SUCCESS);
5902 break;
5903 }
5904
5905 case IEMXCPTRAISE_PREV_EVENT:
5906 {
5907 uint32_t u32ErrCode;
5908 if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
5909 {
5910 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5911 AssertRCReturn(rc2, rc2);
5912 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5913 }
5914 else
5915 u32ErrCode = 0;
5916
5917 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
5918 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5919 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5920 0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
5921
5922 Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
5923 pVCpu->hm.s.Event.u32ErrCode));
5924 Assert(rcStrict == VINF_SUCCESS);
5925 break;
5926 }
5927
5928 case IEMXCPTRAISE_REEXEC_INSTR:
5929 Assert(rcStrict == VINF_SUCCESS);
5930 break;
5931
5932 case IEMXCPTRAISE_DOUBLE_FAULT:
5933 {
5934 /*
5935 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
5936 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
5937 */
5938 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
5939 {
5940 pVmxTransient->fVectoringDoublePF = true;
5941 Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
5942 pVCpu->cpum.GstCtx.cr2));
5943 rcStrict = VINF_SUCCESS;
5944 }
5945 else
5946 {
5947 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5948 hmR0VmxSetPendingXcptDF(pVCpu);
5949 Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
5950 uIdtVector, uExitVector));
5951 rcStrict = VINF_HM_DOUBLE_FAULT;
5952 }
5953 break;
5954 }
5955
5956 case IEMXCPTRAISE_TRIPLE_FAULT:
5957 {
5958 Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
5959 rcStrict = VINF_EM_RESET;
5960 break;
5961 }
5962
5963 case IEMXCPTRAISE_CPU_HANG:
5964 {
5965 Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
5966 rcStrict = VERR_EM_GUEST_CPU_HANG;
5967 break;
5968 }
5969
5970 default:
5971 {
5972 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
5973 rcStrict = VERR_VMX_IPE_2;
5974 break;
5975 }
5976 }
5977 }
5978 else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5979 && VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5980 && uExitVector != X86_XCPT_DF
5981 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
5982 {
5983 /*
5984 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5985 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5986 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5987 */
5988 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5989 {
5990 Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
5991 VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5992 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5993 }
5994 }
5995
5996 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
5997 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
5998 return rcStrict;
5999}
6000
6001
6002/**
6003 * Imports a guest segment register from the current VMCS into
6004 * the guest-CPU context.
6005 *
6006 * @returns VBox status code.
6007 * @param pVCpu The cross context virtual CPU structure.
6008 * @param idxSel Index of the selector in the VMCS.
6009 * @param idxLimit Index of the segment limit in the VMCS.
6010 * @param idxBase Index of the segment base in the VMCS.
6011 * @param idxAccess Index of the access rights of the segment in the VMCS.
6012 * @param pSelReg Pointer to the segment selector.
6013 *
6014 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6015 * do not log!
6016 *
6017 * @remarks Never call this function directly!!! Use the
6018 * HMVMX_IMPORT_SREG() macro as that takes care
6019 * of whether to read from the VMCS cache or not.
6020 */
6021static int hmR0VmxImportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6022 PCPUMSELREG pSelReg)
6023{
6024 NOREF(pVCpu);
6025
6026 uint32_t u32Sel;
6027 uint32_t u32Limit;
6028 uint32_t u32Attr;
6029 uint64_t u64Base;
6030 int rc = VMXReadVmcs32(idxSel, &u32Sel);
6031 rc |= VMXReadVmcs32(idxLimit, &u32Limit);
6032 rc |= VMXReadVmcs32(idxAccess, &u32Attr);
6033 rc |= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
6034 AssertRCReturn(rc, rc);
6035
6036 pSelReg->Sel = (uint16_t)u32Sel;
6037 pSelReg->ValidSel = (uint16_t)u32Sel;
6038 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6039 pSelReg->u32Limit = u32Limit;
6040 pSelReg->u64Base = u64Base;
6041 pSelReg->Attr.u = u32Attr;
6042
6043 /*
6044 * If VT-x marks the segment as unusable, most other bits remain undefined:
6045 * - For CS the L, D and G bits have meaning.
6046 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6047 * - For the remaining data segments no bits are defined.
6048 *
6049 * The present bit and the unusable bit has been observed to be set at the
6050 * same time (the selector was supposed to be invalid as we started executing
6051 * a V8086 interrupt in ring-0).
6052 *
6053 * What should be important for the rest of the VBox code, is that the P bit is
6054 * cleared. Some of the other VBox code recognizes the unusable bit, but
6055 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6056 * safe side here, we'll strip off P and other bits we don't care about. If
6057 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6058 *
6059 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6060 */
6061 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6062 {
6063 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6064
6065 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6066 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6067 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6068#ifdef VBOX_STRICT
6069 VMMRZCallRing3Disable(pVCpu);
6070 Log4Func(("Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Sel, pSelReg->Attr.u));
6071# ifdef DEBUG_bird
6072 AssertMsg((u32Attr & ~X86DESCATTR_P) == pSelReg->Attr.u,
6073 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6074 idxSel, u32Sel, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6075# endif
6076 VMMRZCallRing3Enable(pVCpu);
6077#endif
6078 }
6079 return VINF_SUCCESS;
6080}
6081
6082
6083/**
6084 * Imports the guest RIP from the VMCS back into the guest-CPU context.
6085 *
6086 * @returns VBox status code.
6087 * @param pVCpu The cross context virtual CPU structure.
6088 *
6089 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6090 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6091 * instead!!!
6092 */
6093DECLINLINE(int) hmR0VmxImportGuestRip(PVMCPU pVCpu)
6094{
6095 uint64_t u64Val;
6096 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6097 if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
6098 {
6099 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6100 if (RT_SUCCESS(rc))
6101 {
6102 pCtx->rip = u64Val;
6103 EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
6104 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
6105 }
6106 return rc;
6107 }
6108 return VINF_SUCCESS;
6109}
6110
6111
6112/**
6113 * Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
6114 *
6115 * @returns VBox status code.
6116 * @param pVCpu The cross context virtual CPU structure.
6117 *
6118 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6119 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6120 * instead!!!
6121 */
6122DECLINLINE(int) hmR0VmxImportGuestRFlags(PVMCPU pVCpu)
6123{
6124 uint32_t u32Val;
6125 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6126 if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
6127 {
6128 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
6129 if (RT_SUCCESS(rc))
6130 {
6131 pCtx->eflags.u32 = u32Val;
6132
6133 /* Restore eflags for real-on-v86-mode hack. */
6134 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6135 {
6136 pCtx->eflags.Bits.u1VM = 0;
6137 pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6138 }
6139 }
6140 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
6141 return rc;
6142 }
6143 return VINF_SUCCESS;
6144}
6145
6146
6147/**
6148 * Imports the guest interruptibility-state from the VMCS back into the guest-CPU
6149 * context.
6150 *
6151 * @returns VBox status code.
6152 * @param pVCpu The cross context virtual CPU structure.
6153 *
6154 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6155 * do not log!
6156 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6157 * instead!!!
6158 */
6159DECLINLINE(int) hmR0VmxImportGuestIntrState(PVMCPU pVCpu)
6160{
6161 uint32_t u32Val;
6162 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6163 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
6164 if (RT_SUCCESS(rc))
6165 {
6166 /*
6167 * We additionally have a requirement to import RIP, RFLAGS depending on whether we
6168 * might need them in hmR0VmxEvaluatePendingEvent().
6169 */
6170 if (!u32Val)
6171 {
6172 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6173 {
6174 rc = hmR0VmxImportGuestRip(pVCpu);
6175 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6176 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6177 }
6178
6179 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6180 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6181 }
6182 else
6183 {
6184 rc = hmR0VmxImportGuestRip(pVCpu);
6185 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6186
6187 if (u32Val & ( VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
6188 | VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
6189 {
6190 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
6191 }
6192 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6193 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6194
6195 if (u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)
6196 {
6197 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6198 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6199 }
6200 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6201 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6202 }
6203 }
6204 return rc;
6205}
6206
6207
6208/**
6209 * Worker for VMXR0ImportStateOnDemand.
6210 *
6211 * @returns VBox status code.
6212 * @param pVCpu The cross context virtual CPU structure.
6213 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6214 */
6215static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
6216{
6217#define VMXLOCAL_BREAK_RC(a_rc) \
6218 if (RT_FAILURE(a_rc)) \
6219 break
6220
6221 int rc = VINF_SUCCESS;
6222 PVM pVM = pVCpu->CTX_SUFF(pVM);
6223 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6224 uint64_t u64Val;
6225 uint32_t u32Val;
6226
6227 Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
6228 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
6229
6230 /*
6231 * We disable interrupts to make the updating of the state and in particular
6232 * the fExtrn modification atomic wrt to preemption hooks.
6233 */
6234 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
6235
6236 fWhat &= pCtx->fExtrn;
6237 if (fWhat)
6238 {
6239 do
6240 {
6241 if (fWhat & CPUMCTX_EXTRN_RIP)
6242 {
6243 rc = hmR0VmxImportGuestRip(pVCpu);
6244 VMXLOCAL_BREAK_RC(rc);
6245 }
6246
6247 if (fWhat & CPUMCTX_EXTRN_RFLAGS)
6248 {
6249 rc = hmR0VmxImportGuestRFlags(pVCpu);
6250 VMXLOCAL_BREAK_RC(rc);
6251 }
6252
6253 if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
6254 {
6255 rc = hmR0VmxImportGuestIntrState(pVCpu);
6256 VMXLOCAL_BREAK_RC(rc);
6257 }
6258
6259 if (fWhat & CPUMCTX_EXTRN_RSP)
6260 {
6261 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6262 VMXLOCAL_BREAK_RC(rc);
6263 pCtx->rsp = u64Val;
6264 }
6265
6266 if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
6267 {
6268 if (fWhat & CPUMCTX_EXTRN_CS)
6269 {
6270 rc = HMVMX_IMPORT_SREG(CS, &pCtx->cs);
6271 rc |= hmR0VmxImportGuestRip(pVCpu);
6272 VMXLOCAL_BREAK_RC(rc);
6273 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6274 pCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6275 EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true);
6276 }
6277 if (fWhat & CPUMCTX_EXTRN_SS)
6278 {
6279 rc = HMVMX_IMPORT_SREG(SS, &pCtx->ss);
6280 VMXLOCAL_BREAK_RC(rc);
6281 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6282 pCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6283 }
6284 if (fWhat & CPUMCTX_EXTRN_DS)
6285 {
6286 rc = HMVMX_IMPORT_SREG(DS, &pCtx->ds);
6287 VMXLOCAL_BREAK_RC(rc);
6288 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6289 pCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6290 }
6291 if (fWhat & CPUMCTX_EXTRN_ES)
6292 {
6293 rc = HMVMX_IMPORT_SREG(ES, &pCtx->es);
6294 VMXLOCAL_BREAK_RC(rc);
6295 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6296 pCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6297 }
6298 if (fWhat & CPUMCTX_EXTRN_FS)
6299 {
6300 rc = HMVMX_IMPORT_SREG(FS, &pCtx->fs);
6301 VMXLOCAL_BREAK_RC(rc);
6302 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6303 pCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6304 }
6305 if (fWhat & CPUMCTX_EXTRN_GS)
6306 {
6307 rc = HMVMX_IMPORT_SREG(GS, &pCtx->gs);
6308 VMXLOCAL_BREAK_RC(rc);
6309 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6310 pCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6311 }
6312 }
6313
6314 if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
6315 {
6316 if (fWhat & CPUMCTX_EXTRN_LDTR)
6317 {
6318 rc = HMVMX_IMPORT_SREG(LDTR, &pCtx->ldtr);
6319 VMXLOCAL_BREAK_RC(rc);
6320 }
6321
6322 if (fWhat & CPUMCTX_EXTRN_GDTR)
6323 {
6324 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6325 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
6326 VMXLOCAL_BREAK_RC(rc);
6327 pCtx->gdtr.pGdt = u64Val;
6328 pCtx->gdtr.cbGdt = u32Val;
6329 }
6330
6331 /* Guest IDTR. */
6332 if (fWhat & CPUMCTX_EXTRN_IDTR)
6333 {
6334 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6335 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
6336 VMXLOCAL_BREAK_RC(rc);
6337 pCtx->idtr.pIdt = u64Val;
6338 pCtx->idtr.cbIdt = u32Val;
6339 }
6340
6341 /* Guest TR. */
6342 if (fWhat & CPUMCTX_EXTRN_TR)
6343 {
6344 /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, don't save that one. */
6345 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6346 {
6347 rc = HMVMX_IMPORT_SREG(TR, &pCtx->tr);
6348 VMXLOCAL_BREAK_RC(rc);
6349 }
6350 }
6351 }
6352
6353 if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
6354 {
6355 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
6356 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
6357 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
6358 pCtx->SysEnter.cs = u32Val;
6359 VMXLOCAL_BREAK_RC(rc);
6360 }
6361
6362#if HC_ARCH_BITS == 64
6363 if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
6364 {
6365 if ( pVM->hm.s.fAllow64BitGuests
6366 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6367 pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
6368 }
6369
6370 if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
6371 {
6372 if ( pVM->hm.s.fAllow64BitGuests
6373 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6374 {
6375 pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
6376 pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
6377 pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
6378 }
6379 }
6380#endif
6381
6382 if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
6383#if HC_ARCH_BITS == 32
6384 || (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS))
6385#endif
6386 )
6387 {
6388 PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6389 uint32_t const cMsrs = pVCpu->hm.s.vmx.cMsrs;
6390 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6391 {
6392 switch (pMsr->u32Msr)
6393 {
6394#if HC_ARCH_BITS == 32
6395 case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
6396 case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
6397 case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
6398 case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6399#endif
6400 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6401 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6402 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit */ break;
6403 default:
6404 {
6405 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6406 ASMSetFlags(fEFlags);
6407 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
6408 cMsrs));
6409 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6410 }
6411 }
6412 }
6413 }
6414
6415 if (fWhat & CPUMCTX_EXTRN_DR7)
6416 {
6417 if (!pVCpu->hm.s.fUsingHyperDR7)
6418 {
6419 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6420 rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
6421 VMXLOCAL_BREAK_RC(rc);
6422 pCtx->dr[7] = u32Val;
6423 }
6424 }
6425
6426 if (fWhat & CPUMCTX_EXTRN_CR_MASK)
6427 {
6428 uint32_t u32Shadow;
6429 if (fWhat & CPUMCTX_EXTRN_CR0)
6430 {
6431 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
6432 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
6433 VMXLOCAL_BREAK_RC(rc);
6434 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr0Mask)
6435 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr0Mask);
6436 VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
6437 CPUMSetGuestCR0(pVCpu, u32Val);
6438 VMMRZCallRing3Enable(pVCpu);
6439 }
6440
6441 if (fWhat & CPUMCTX_EXTRN_CR4)
6442 {
6443 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
6444 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
6445 VMXLOCAL_BREAK_RC(rc);
6446 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr4Mask)
6447 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr4Mask);
6448 CPUMSetGuestCR4(pVCpu, u32Val);
6449 }
6450
6451 if (fWhat & CPUMCTX_EXTRN_CR3)
6452 {
6453 /* CR0.PG bit changes are always intercepted, so it's up to date. */
6454 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6455 || ( pVM->hm.s.fNestedPaging
6456 && CPUMIsGuestPagingEnabledEx(pCtx)))
6457 {
6458 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6459 if (pCtx->cr3 != u64Val)
6460 {
6461 CPUMSetGuestCR3(pVCpu, u64Val);
6462 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6463 }
6464
6465 /* If the guest is in PAE mode, sync back the PDPE's into the guest state.
6466 Note: CR4.PAE, CR0.PG, EFER bit changes are always intercepted, so they're up to date. */
6467 if (CPUMIsGuestInPAEModeEx(pCtx))
6468 {
6469 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6470 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6471 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6472 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6473 VMXLOCAL_BREAK_RC(rc);
6474 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6475 }
6476 }
6477 }
6478 }
6479 } while (0);
6480
6481 if (RT_SUCCESS(rc))
6482 {
6483 /* Update fExtrn. */
6484 pCtx->fExtrn &= ~fWhat;
6485
6486 /* If everything has been imported, clear the HM keeper bit. */
6487 if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
6488 {
6489 pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
6490 Assert(!pCtx->fExtrn);
6491 }
6492 }
6493 }
6494 else
6495 AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
6496
6497 ASMSetFlags(fEFlags);
6498
6499 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
6500
6501 /*
6502 * Honor any pending CR3 updates.
6503 *
6504 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6505 * -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6506 * -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
6507 *
6508 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6509 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6510 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6511 * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
6512 *
6513 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6514 */
6515 if (VMMRZCallRing3IsEnabled(pVCpu))
6516 {
6517 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6518 {
6519 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
6520 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6521 }
6522
6523 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6524 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6525
6526 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6527 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6528 }
6529
6530 return VINF_SUCCESS;
6531#undef VMXLOCAL_BREAK_RC
6532}
6533
6534
6535/**
6536 * Saves the guest state from the VMCS into the guest-CPU context.
6537 *
6538 * @returns VBox status code.
6539 * @param pVCpu The cross context virtual CPU structure.
6540 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6541 */
6542VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
6543{
6544 return hmR0VmxImportGuestState(pVCpu, fWhat);
6545}
6546
6547
6548/**
6549 * Check per-VM and per-VCPU force flag actions that require us to go back to
6550 * ring-3 for one reason or another.
6551 *
6552 * @returns Strict VBox status code (i.e. informational status codes too)
6553 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6554 * ring-3.
6555 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6556 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6557 * interrupts)
6558 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6559 * all EMTs to be in ring-3.
6560 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6561 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6562 * to the EM loop.
6563 *
6564 * @param pVCpu The cross context virtual CPU structure.
6565 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6566 */
6567static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
6568{
6569 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6570
6571 /*
6572 * Anything pending? Should be more likely than not if we're doing a good job.
6573 */
6574 PVM pVM = pVCpu->CTX_SUFF(pVM);
6575 if ( !fStepping
6576 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6577 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6578 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6579 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6580 return VINF_SUCCESS;
6581
6582 /* Pending PGM C3 sync. */
6583 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6584 {
6585 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6586 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4)));
6587 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
6588 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6589 if (rcStrict2 != VINF_SUCCESS)
6590 {
6591 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6592 Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6593 return rcStrict2;
6594 }
6595 }
6596
6597 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6598 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6599 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6600 {
6601 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6602 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6603 Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6604 return rc2;
6605 }
6606
6607 /* Pending VM request packets, such as hardware interrupts. */
6608 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6609 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6610 {
6611 Log4Func(("Pending VM request forcing us back to ring-3\n"));
6612 return VINF_EM_PENDING_REQUEST;
6613 }
6614
6615 /* Pending PGM pool flushes. */
6616 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6617 {
6618 Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
6619 return VINF_PGM_POOL_FLUSH_PENDING;
6620 }
6621
6622 /* Pending DMA requests. */
6623 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6624 {
6625 Log4Func(("Pending DMA request forcing us back to ring-3\n"));
6626 return VINF_EM_RAW_TO_R3;
6627 }
6628
6629 return VINF_SUCCESS;
6630}
6631
6632
6633/**
6634 * Converts any TRPM trap into a pending HM event. This is typically used when
6635 * entering from ring-3 (not longjmp returns).
6636 *
6637 * @param pVCpu The cross context virtual CPU structure.
6638 */
6639static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6640{
6641 Assert(TRPMHasTrap(pVCpu));
6642 Assert(!pVCpu->hm.s.Event.fPending);
6643
6644 uint8_t uVector;
6645 TRPMEVENT enmTrpmEvent;
6646 RTGCUINT uErrCode;
6647 RTGCUINTPTR GCPtrFaultAddress;
6648 uint8_t cbInstr;
6649
6650 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6651 AssertRC(rc);
6652
6653 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6654 uint32_t u32IntInfo = uVector | VMX_EXIT_INT_INFO_VALID;
6655 if (enmTrpmEvent == TRPM_TRAP)
6656 {
6657 switch (uVector)
6658 {
6659 case X86_XCPT_NMI:
6660 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
6661 break;
6662
6663 case X86_XCPT_BP:
6664 case X86_XCPT_OF:
6665 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
6666 break;
6667
6668 case X86_XCPT_PF:
6669 case X86_XCPT_DF:
6670 case X86_XCPT_TS:
6671 case X86_XCPT_NP:
6672 case X86_XCPT_SS:
6673 case X86_XCPT_GP:
6674 case X86_XCPT_AC:
6675 u32IntInfo |= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
6676 RT_FALL_THRU();
6677 default:
6678 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
6679 break;
6680 }
6681 }
6682 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6683 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
6684 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6685 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
6686 else
6687 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6688
6689 rc = TRPMResetTrap(pVCpu);
6690 AssertRC(rc);
6691 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6692 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6693
6694 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6695}
6696
6697
6698/**
6699 * Converts the pending HM event into a TRPM trap.
6700 *
6701 * @param pVCpu The cross context virtual CPU structure.
6702 */
6703static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6704{
6705 Assert(pVCpu->hm.s.Event.fPending);
6706
6707 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6708 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6709 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
6710 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6711
6712 /* If a trap was already pending, we did something wrong! */
6713 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6714
6715 TRPMEVENT enmTrapType;
6716 switch (uVectorType)
6717 {
6718 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6719 enmTrapType = TRPM_HARDWARE_INT;
6720 break;
6721
6722 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6723 enmTrapType = TRPM_SOFTWARE_INT;
6724 break;
6725
6726 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6727 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6728 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6729 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6730 enmTrapType = TRPM_TRAP;
6731 break;
6732
6733 default:
6734 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6735 enmTrapType = TRPM_32BIT_HACK;
6736 break;
6737 }
6738
6739 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6740
6741 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6742 AssertRC(rc);
6743
6744 if (fErrorCodeValid)
6745 TRPMSetErrorCode(pVCpu, uErrorCode);
6746
6747 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6748 && uVector == X86_XCPT_PF)
6749 {
6750 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6751 }
6752 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6753 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6754 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6755 {
6756 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6757 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6758 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6759 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6760 }
6761
6762 /* Clear any pending events from the VMCS. */
6763 VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
6764 VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, 0);
6765
6766 /* We're now done converting the pending event. */
6767 pVCpu->hm.s.Event.fPending = false;
6768}
6769
6770
6771/**
6772 * Does the necessary state syncing before returning to ring-3 for any reason
6773 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6774 *
6775 * @returns VBox status code.
6776 * @param pVCpu The cross context virtual CPU structure.
6777 * @param fImportState Whether to import the guest state from the VMCS back
6778 * to the guest-CPU context.
6779 *
6780 * @remarks No-long-jmp zone!!!
6781 */
6782static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
6783{
6784 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6785 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6786
6787 RTCPUID idCpu = RTMpCpuId();
6788 Log4Func(("HostCpuId=%u\n", idCpu));
6789
6790 /*
6791 * !!! IMPORTANT !!!
6792 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
6793 */
6794
6795 /* Save the guest state if necessary. */
6796 if (fImportState)
6797 {
6798 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
6799 AssertRCReturn(rc, rc);
6800 }
6801
6802 /* Restore host FPU state if necessary. We will resync on next R0 reentry. */
6803 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
6804 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6805
6806 /* Restore host debug registers if necessary. We will resync on next R0 reentry. */
6807#ifdef VBOX_STRICT
6808 if (CPUMIsHyperDebugStateActive(pVCpu))
6809 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
6810#endif
6811 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6812 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6813 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6814
6815#if HC_ARCH_BITS == 64
6816 /* Restore host-state bits that VT-x only restores partially. */
6817 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6818 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6819 {
6820 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6821 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6822 }
6823 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6824#endif
6825
6826 /* Restore the lazy host MSRs as we're leaving VT-x context. */
6827 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
6828 {
6829 /* We shouldn't restore the host MSRs without saving the guest MSRs first. */
6830 if (!fImportState)
6831 {
6832 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS);
6833 AssertRCReturn(rc, rc);
6834 }
6835 hmR0VmxLazyRestoreHostMsrs(pVCpu);
6836 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
6837 }
6838 else
6839 pVCpu->hm.s.vmx.fLazyMsrs = 0;
6840
6841 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6842 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6843
6844 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6845 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
6846 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
6847 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
6848 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
6849 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6850 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6851 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6852 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6853
6854 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6855
6856 /** @todo This partially defeats the purpose of having preemption hooks.
6857 * The problem is, deregistering the hooks should be moved to a place that
6858 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6859 * context.
6860 */
6861 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6862 {
6863 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6864 AssertRCReturn(rc, rc);
6865
6866 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6867 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6868 }
6869 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6870 NOREF(idCpu);
6871
6872 return VINF_SUCCESS;
6873}
6874
6875
6876/**
6877 * Leaves the VT-x session.
6878 *
6879 * @returns VBox status code.
6880 * @param pVCpu The cross context virtual CPU structure.
6881 *
6882 * @remarks No-long-jmp zone!!!
6883 */
6884static int hmR0VmxLeaveSession(PVMCPU pVCpu)
6885{
6886 HM_DISABLE_PREEMPT(pVCpu);
6887 HMVMX_ASSERT_CPU_SAFE(pVCpu);
6888 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6889 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6890
6891 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6892 and done this from the VMXR0ThreadCtxCallback(). */
6893 if (!pVCpu->hm.s.fLeaveDone)
6894 {
6895 int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
6896 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
6897 pVCpu->hm.s.fLeaveDone = true;
6898 }
6899 Assert(!pVCpu->cpum.GstCtx.fExtrn);
6900
6901 /*
6902 * !!! IMPORTANT !!!
6903 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6904 */
6905
6906 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6907 /** @todo Deregistering here means we need to VMCLEAR always
6908 * (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
6909 * for calling VMMR0ThreadCtxHookDisable here! */
6910 VMMR0ThreadCtxHookDisable(pVCpu);
6911
6912 /* Leave HM context. This takes care of local init (term). */
6913 int rc = HMR0LeaveCpu(pVCpu);
6914
6915 HM_RESTORE_PREEMPT();
6916 return rc;
6917}
6918
6919
6920/**
6921 * Does the necessary state syncing before doing a longjmp to ring-3.
6922 *
6923 * @returns VBox status code.
6924 * @param pVCpu The cross context virtual CPU structure.
6925 *
6926 * @remarks No-long-jmp zone!!!
6927 */
6928DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
6929{
6930 return hmR0VmxLeaveSession(pVCpu);
6931}
6932
6933
6934/**
6935 * Take necessary actions before going back to ring-3.
6936 *
6937 * An action requires us to go back to ring-3. This function does the necessary
6938 * steps before we can safely return to ring-3. This is not the same as longjmps
6939 * to ring-3, this is voluntary and prepares the guest so it may continue
6940 * executing outside HM (recompiler/IEM).
6941 *
6942 * @returns VBox status code.
6943 * @param pVCpu The cross context virtual CPU structure.
6944 * @param rcExit The reason for exiting to ring-3. Can be
6945 * VINF_VMM_UNKNOWN_RING3_CALL.
6946 */
6947static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
6948{
6949 Assert(pVCpu);
6950 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6951
6952 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6953 {
6954 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6955 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
6956 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6957 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6958 }
6959
6960 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6961 VMMRZCallRing3Disable(pVCpu);
6962 Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
6963
6964 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6965 if (pVCpu->hm.s.Event.fPending)
6966 {
6967 hmR0VmxPendingEventToTrpmTrap(pVCpu);
6968 Assert(!pVCpu->hm.s.Event.fPending);
6969 }
6970
6971 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
6972 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
6973
6974 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
6975 and if we're injecting an event we should have a TRPM trap pending. */
6976 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6977#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
6978 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6979#endif
6980
6981 /* Save guest state and restore host state bits. */
6982 int rc = hmR0VmxLeaveSession(pVCpu);
6983 AssertRCReturn(rc, rc);
6984 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6985 /* Thread-context hooks are unregistered at this point!!! */
6986
6987 /* Sync recompiler state. */
6988 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6989 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6990 | CPUM_CHANGED_LDTR
6991 | CPUM_CHANGED_GDTR
6992 | CPUM_CHANGED_IDTR
6993 | CPUM_CHANGED_TR
6994 | CPUM_CHANGED_HIDDEN_SEL_REGS);
6995 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
6996 && CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
6997 {
6998 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6999 }
7000
7001 Assert(!pVCpu->hm.s.fClearTrapFlag);
7002
7003 /* Update the exit-to-ring 3 reason. */
7004 pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
7005
7006 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7007 if (rcExit != VINF_EM_RAW_INTERRUPT)
7008 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7009
7010 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7011
7012 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7013 VMMRZCallRing3RemoveNotification(pVCpu);
7014 VMMRZCallRing3Enable(pVCpu);
7015
7016 return rc;
7017}
7018
7019
7020/**
7021 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7022 * longjump to ring-3 and possibly get preempted.
7023 *
7024 * @returns VBox status code.
7025 * @param pVCpu The cross context virtual CPU structure.
7026 * @param enmOperation The operation causing the ring-3 longjump.
7027 * @param pvUser User argument, currently unused, NULL.
7028 */
7029static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7030{
7031 RT_NOREF(pvUser);
7032 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7033 {
7034 /*
7035 * !!! IMPORTANT !!!
7036 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7037 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7038 */
7039 VMMRZCallRing3RemoveNotification(pVCpu);
7040 VMMRZCallRing3Disable(pVCpu);
7041 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7042 RTThreadPreemptDisable(&PreemptState);
7043
7044 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
7045 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7046 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7047
7048#if HC_ARCH_BITS == 64
7049 /* Restore host-state bits that VT-x only restores partially. */
7050 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7051 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7052 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7053 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7054#endif
7055
7056 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7057 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7058 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7059
7060 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7061 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7062 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7063 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7064 {
7065 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7066 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7067 }
7068
7069 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7070 VMMR0ThreadCtxHookDisable(pVCpu);
7071 HMR0LeaveCpu(pVCpu);
7072 RTThreadPreemptRestore(&PreemptState);
7073 return VINF_SUCCESS;
7074 }
7075
7076 Assert(pVCpu);
7077 Assert(pvUser);
7078 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7079 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7080
7081 VMMRZCallRing3Disable(pVCpu);
7082 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7083
7084 Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
7085
7086 int rc = hmR0VmxLongJmpToRing3(pVCpu);
7087 AssertRCReturn(rc, rc);
7088
7089 VMMRZCallRing3Enable(pVCpu);
7090 return VINF_SUCCESS;
7091}
7092
7093
7094/**
7095 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7096 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7097 *
7098 * @param pVCpu The cross context virtual CPU structure.
7099 */
7100DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7101{
7102 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7103 {
7104 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7105 {
7106 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_PROC_CTLS_INT_WINDOW_EXIT;
7107 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7108 AssertRC(rc);
7109 Log4Func(("Setup interrupt-window exiting\n"));
7110 }
7111 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7112}
7113
7114
7115/**
7116 * Clears the interrupt-window exiting control in the VMCS.
7117 *
7118 * @param pVCpu The cross context virtual CPU structure.
7119 */
7120DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7121{
7122 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT);
7123 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
7124 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7125 AssertRC(rc);
7126 Log4Func(("Cleared interrupt-window exiting\n"));
7127}
7128
7129
7130/**
7131 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7132 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7133 *
7134 * @param pVCpu The cross context virtual CPU structure.
7135 */
7136DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7137{
7138 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7139 {
7140 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7141 {
7142 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7143 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7144 AssertRC(rc);
7145 Log4Func(("Setup NMI-window exiting\n"));
7146 }
7147 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7148}
7149
7150
7151/**
7152 * Clears the NMI-window exiting control in the VMCS.
7153 *
7154 * @param pVCpu The cross context virtual CPU structure.
7155 */
7156DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7157{
7158 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT);
7159 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7160 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7161 AssertRC(rc);
7162 Log4Func(("Cleared NMI-window exiting\n"));
7163}
7164
7165
7166/**
7167 * Evaluates the event to be delivered to the guest and sets it as the pending
7168 * event.
7169 *
7170 * @returns The VT-x guest-interruptibility state.
7171 * @param pVCpu The cross context virtual CPU structure.
7172 */
7173static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu)
7174{
7175 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7176 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7177 uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu);
7178 bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7179 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7180 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7181
7182 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7183 Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7184 Assert(!fBlockSti || pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7185 Assert(!TRPMHasTrap(pVCpu));
7186
7187 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7188 APICUpdatePendingInterrupts(pVCpu);
7189
7190 /*
7191 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7192 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7193 */
7194 /** @todo SMI. SMIs take priority over NMIs. */
7195 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7196 {
7197 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7198 if ( !pVCpu->hm.s.Event.fPending
7199 && !fBlockNmi
7200 && !fBlockSti
7201 && !fBlockMovSS)
7202 {
7203 Log4Func(("Pending NMI\n"));
7204 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INT_INFO_VALID;
7205 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7206
7207 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7208 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7209 }
7210 else
7211 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7212 }
7213 /*
7214 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7215 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7216 */
7217 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7218 && !pVCpu->hm.s.fSingleInstruction)
7219 {
7220 Assert(!DBGFIsStepping(pVCpu));
7221 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7222 AssertRCReturn(rc, 0);
7223 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7224 if ( !pVCpu->hm.s.Event.fPending
7225 && !fBlockInt
7226 && !fBlockSti
7227 && !fBlockMovSS)
7228 {
7229 uint8_t u8Interrupt;
7230 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7231 if (RT_SUCCESS(rc))
7232 {
7233 Log4Func(("Pending external interrupt u8Interrupt=%#x\n", u8Interrupt));
7234 uint32_t u32IntInfo = u8Interrupt
7235 | VMX_EXIT_INT_INFO_VALID
7236 | (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7237
7238 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7239 }
7240 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7241 {
7242 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
7243 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7244 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7245
7246 /*
7247 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7248 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7249 * need to re-set this force-flag here.
7250 */
7251 }
7252 else
7253 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7254 }
7255 else
7256 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7257 }
7258
7259 return fIntrState;
7260}
7261
7262
7263/**
7264 * Sets a pending-debug exception to be delivered to the guest if the guest is
7265 * single-stepping in the VMCS.
7266 *
7267 * @param pVCpu The cross context virtual CPU structure.
7268 */
7269DECLINLINE(int) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7270{
7271 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7272 RT_NOREF(pVCpu);
7273 return VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
7274}
7275
7276
7277/**
7278 * Injects any pending events into the guest if the guest is in a state to
7279 * receive them.
7280 *
7281 * @returns Strict VBox status code (i.e. informational status codes too).
7282 * @param pVCpu The cross context virtual CPU structure.
7283 * @param fIntrState The VT-x guest-interruptibility state.
7284 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7285 * return VINF_EM_DBG_STEPPED if the event was
7286 * dispatched directly.
7287 */
7288static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, uint32_t fIntrState, bool fStepping)
7289{
7290 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7291 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7292
7293 bool fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7294 bool fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7295
7296 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7297 Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7298 Assert(!fBlockSti || pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7299 Assert(!TRPMHasTrap(pVCpu));
7300
7301 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7302 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7303 if (pVCpu->hm.s.Event.fPending)
7304 {
7305 /*
7306 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7307 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7308 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7309 *
7310 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7311 */
7312 uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7313#ifdef VBOX_STRICT
7314 if (uIntType == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
7315 {
7316 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7317 Assert(!fBlockInt);
7318 Assert(!fBlockSti);
7319 Assert(!fBlockMovSS);
7320 }
7321 else if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
7322 {
7323 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7324 Assert(!fBlockSti);
7325 Assert(!fBlockMovSS);
7326 Assert(!fBlockNmi);
7327 }
7328#endif
7329 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7330 uIntType));
7331 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7332 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping,
7333 &fIntrState);
7334 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7335
7336 /* Update the interruptibility-state as it could have been changed by
7337 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7338 fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7339 fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7340
7341 if (uIntType == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
7342 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7343 else
7344 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7345 }
7346
7347 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7348 if ( fBlockSti
7349 || fBlockMovSS)
7350 {
7351 if (!pVCpu->hm.s.fSingleInstruction)
7352 {
7353 /*
7354 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7355 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7356 * See Intel spec. 27.3.4 "Saving Non-Register State".
7357 */
7358 Assert(!DBGFIsStepping(pVCpu));
7359 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7360 AssertRCReturn(rc, rc);
7361 if (pCtx->eflags.Bits.u1TF)
7362 {
7363 int rc2 = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7364 AssertRCReturn(rc2, rc2);
7365 }
7366 }
7367 else if (pCtx->eflags.Bits.u1TF)
7368 {
7369 /*
7370 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7371 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7372 */
7373 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
7374 fIntrState = 0;
7375 }
7376 }
7377
7378 /*
7379 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7380 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7381 */
7382 int rc3 = hmR0VmxExportGuestIntrState(pVCpu, fIntrState);
7383 AssertRCReturn(rc3, rc3);
7384
7385 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7386 NOREF(fBlockMovSS); NOREF(fBlockSti);
7387 return rcStrict;
7388}
7389
7390
7391/**
7392 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7393 *
7394 * @param pVCpu The cross context virtual CPU structure.
7395 */
7396DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
7397{
7398 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INT_INFO_VALID;
7399 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7400}
7401
7402
7403/**
7404 * Injects a double-fault (\#DF) exception into the VM.
7405 *
7406 * @returns Strict VBox status code (i.e. informational status codes too).
7407 * @param pVCpu The cross context virtual CPU structure.
7408 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7409 * and should return VINF_EM_DBG_STEPPED if the event
7410 * is injected directly (register modified by us, not
7411 * by hardware on VM-entry).
7412 * @param pfIntrState Pointer to the current guest interruptibility-state.
7413 * This interruptibility-state will be updated if
7414 * necessary. This cannot not be NULL.
7415 */
7416DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, bool fStepping, uint32_t *pfIntrState)
7417{
7418 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INT_INFO_VALID;
7419 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7420 u32IntInfo |= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7421 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */, fStepping,
7422 pfIntrState);
7423}
7424
7425
7426/**
7427 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7428 *
7429 * @param pVCpu The cross context virtual CPU structure.
7430 */
7431DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
7432{
7433 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INT_INFO_VALID;
7434 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7435 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7436}
7437
7438
7439/**
7440 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7441 *
7442 * @param pVCpu The cross context virtual CPU structure.
7443 * @param cbInstr The value of RIP that is to be pushed on the guest
7444 * stack.
7445 */
7446DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, uint32_t cbInstr)
7447{
7448 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INT_INFO_VALID;
7449 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7450 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7451}
7452
7453
7454/**
7455 * Injects a general-protection (\#GP) fault into the VM.
7456 *
7457 * @returns Strict VBox status code (i.e. informational status codes too).
7458 * @param pVCpu The cross context virtual CPU structure.
7459 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7460 * mode, i.e. in real-mode it's not valid).
7461 * @param u32ErrorCode The error code associated with the \#GP.
7462 * @param fStepping Whether we're running in
7463 * hmR0VmxRunGuestCodeStep() and should return
7464 * VINF_EM_DBG_STEPPED if the event is injected
7465 * directly (register modified by us, not by
7466 * hardware on VM-entry).
7467 * @param pfIntrState Pointer to the current guest interruptibility-state.
7468 * This interruptibility-state will be updated if
7469 * necessary. This cannot not be NULL.
7470 */
7471DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, bool fErrorCodeValid, uint32_t u32ErrorCode, bool fStepping,
7472 uint32_t *pfIntrState)
7473{
7474 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INT_INFO_VALID;
7475 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7476 if (fErrorCodeValid)
7477 u32IntInfo |= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7478 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */, fStepping,
7479 pfIntrState);
7480}
7481
7482
7483/**
7484 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7485 *
7486 * @param pVCpu The cross context virtual CPU structure.
7487 * @param uVector The software interrupt vector number.
7488 * @param cbInstr The value of RIP that is to be pushed on the guest
7489 * stack.
7490 */
7491DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, uint16_t uVector, uint32_t cbInstr)
7492{
7493 uint32_t u32IntInfo = uVector | VMX_EXIT_INT_INFO_VALID;
7494 if ( uVector == X86_XCPT_BP
7495 || uVector == X86_XCPT_OF)
7496 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7497 else
7498 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7499 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7500}
7501
7502
7503/**
7504 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7505 * stack.
7506 *
7507 * @returns Strict VBox status code (i.e. informational status codes too).
7508 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7509 * @param pVCpu The cross context virtual CPU structure.
7510 * @param uValue The value to push to the guest stack.
7511 */
7512static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
7513{
7514 /*
7515 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7516 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7517 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7518 */
7519 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7520 if (pCtx->sp == 1)
7521 return VINF_EM_RESET;
7522 pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7523 int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
7524 AssertRC(rc);
7525 return rc;
7526}
7527
7528
7529/**
7530 * Injects an event into the guest upon VM-entry by updating the relevant fields
7531 * in the VM-entry area in the VMCS.
7532 *
7533 * @returns Strict VBox status code (i.e. informational status codes too).
7534 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7535 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7536 *
7537 * @param pVCpu The cross context virtual CPU structure.
7538 * @param u64IntInfo The VM-entry interruption-information field.
7539 * @param cbInstr The VM-entry instruction length in bytes (for
7540 * software interrupts, exceptions and privileged
7541 * software exceptions).
7542 * @param u32ErrCode The VM-entry exception error code.
7543 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7544 * @param pfIntrState Pointer to the current guest interruptibility-state.
7545 * This interruptibility-state will be updated if
7546 * necessary. This cannot not be NULL.
7547 * @param fStepping Whether we're running in
7548 * hmR0VmxRunGuestCodeStep() and should return
7549 * VINF_EM_DBG_STEPPED if the event is injected
7550 * directly (register modified by us, not by
7551 * hardware on VM-entry).
7552 */
7553static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7554 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState)
7555{
7556 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7557 AssertMsg(!RT_HI_U32(u64IntInfo), ("%#RX64\n", u64IntInfo));
7558 Assert(pfIntrState);
7559
7560 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7561 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7562 uint32_t const uVector = VMX_EXIT_INT_INFO_VECTOR(u32IntInfo);
7563 uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(u32IntInfo);
7564
7565#ifdef VBOX_STRICT
7566 /*
7567 * Validate the error-code-valid bit for hardware exceptions.
7568 * No error codes for exceptions in real-mode.
7569 *
7570 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7571 */
7572 if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
7573 && !CPUMIsGuestInRealModeEx(pCtx))
7574 {
7575 switch (uVector)
7576 {
7577 case X86_XCPT_PF:
7578 case X86_XCPT_DF:
7579 case X86_XCPT_TS:
7580 case X86_XCPT_NP:
7581 case X86_XCPT_SS:
7582 case X86_XCPT_GP:
7583 case X86_XCPT_AC:
7584 AssertMsg(VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
7585 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7586 RT_FALL_THRU();
7587 default:
7588 break;
7589 }
7590 }
7591#endif
7592
7593 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7594 Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
7595 || !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
7596
7597 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7598
7599 /*
7600 * Hardware interrupts & exceptions cannot be delivered through the software interrupt
7601 * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
7602 * interrupt handler in the (real-mode) guest.
7603 *
7604 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
7605 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7606 */
7607 if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
7608 {
7609 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
7610 {
7611 /*
7612 * For unrestricted execution enabled CPUs running real-mode guests, we must not
7613 * set the deliver-error-code bit.
7614 *
7615 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7616 */
7617 u32IntInfo &= ~VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7618 }
7619 else
7620 {
7621 PVM pVM = pVCpu->CTX_SUFF(pVM);
7622 Assert(PDMVmmDevHeapIsEnabled(pVM));
7623 Assert(pVM->hm.s.vmx.pRealModeTSS);
7624
7625 /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
7626 int rc2 = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_TABLE_MASK | CPUMCTX_EXTRN_RIP
7627 | CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_RFLAGS);
7628 AssertRCReturn(rc2, rc2);
7629
7630 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7631 size_t const cbIdtEntry = sizeof(X86IDTR16);
7632 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
7633 {
7634 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7635 if (uVector == X86_XCPT_DF)
7636 return VINF_EM_RESET;
7637
7638 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7639 if (uVector == X86_XCPT_GP)
7640 return hmR0VmxInjectXcptDF(pVCpu, fStepping, pfIntrState);
7641
7642 /*
7643 * If we're injecting an event with no valid IDT entry, inject a #GP.
7644 * No error codes for exceptions in real-mode.
7645 *
7646 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7647 */
7648 return hmR0VmxInjectXcptGP(pVCpu, false /* fErrCodeValid */, 0 /* u32ErrCode */, fStepping, pfIntrState);
7649 }
7650
7651 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7652 uint16_t uGuestIp = pCtx->ip;
7653 if (uIntType == VMX_EXIT_INT_INFO_TYPE_SW_XCPT)
7654 {
7655 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7656 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7657 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7658 }
7659 else if (uIntType == VMX_EXIT_INT_INFO_TYPE_SW_INT)
7660 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7661
7662 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7663 X86IDTR16 IdtEntry;
7664 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
7665 rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7666 AssertRCReturn(rc2, rc2);
7667
7668 /* Construct the stack frame for the interrupt/exception handler. */
7669 VBOXSTRICTRC rcStrict;
7670 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
7671 if (rcStrict == VINF_SUCCESS)
7672 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
7673 if (rcStrict == VINF_SUCCESS)
7674 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
7675
7676 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7677 if (rcStrict == VINF_SUCCESS)
7678 {
7679 pCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7680 pCtx->rip = IdtEntry.offSel;
7681 pCtx->cs.Sel = IdtEntry.uSel;
7682 pCtx->cs.ValidSel = IdtEntry.uSel;
7683 pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7684 if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
7685 && uVector == X86_XCPT_PF)
7686 pCtx->cr2 = GCPtrFaultAddress;
7687
7688 /* If any other guest-state bits are changed here, make sure to update
7689 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7690 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS | HM_CHANGED_GUEST_CR2
7691 | HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
7692 | HM_CHANGED_GUEST_RSP);
7693
7694 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7695 if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
7696 {
7697 Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
7698 && uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
7699 Log4Func(("Clearing inhibition due to STI\n"));
7700 *pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
7701 }
7702 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7703 u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
7704
7705 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7706 it, if we are returning to ring-3 before executing guest code. */
7707 pVCpu->hm.s.Event.fPending = false;
7708
7709 /* Make hmR0VmxPreRunGuest() return if we're stepping since we've changed cs:rip. */
7710 if (fStepping)
7711 rcStrict = VINF_EM_DBG_STEPPED;
7712 }
7713 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7714 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7715 return rcStrict;
7716 }
7717 }
7718
7719 /* Validate. */
7720 Assert(VMX_EXIT_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7721 Assert(!VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7722 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7723
7724 /* Inject. */
7725 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7726 if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
7727 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7728 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7729 AssertRCReturn(rc, rc);
7730
7731 /* Update CR2. */
7732 if ( VMX_EXIT_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
7733 && uVector == X86_XCPT_PF)
7734 pCtx->cr2 = GCPtrFaultAddress;
7735
7736 Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
7737
7738 return VINF_SUCCESS;
7739}
7740
7741
7742/**
7743 * Clears the interrupt-window exiting control in the VMCS and if necessary
7744 * clears the current event in the VMCS as well.
7745 *
7746 * @returns VBox status code.
7747 * @param pVCpu The cross context virtual CPU structure.
7748 *
7749 * @remarks Use this function only to clear events that have not yet been
7750 * delivered to the guest but are injected in the VMCS!
7751 * @remarks No-long-jump zone!!!
7752 */
7753static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
7754{
7755 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7756 {
7757 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7758 Log4Func(("Cleared interrupt widow\n"));
7759 }
7760
7761 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
7762 {
7763 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
7764 Log4Func(("Cleared interrupt widow\n"));
7765 }
7766}
7767
7768
7769/**
7770 * Enters the VT-x session.
7771 *
7772 * @returns VBox status code.
7773 * @param pVCpu The cross context virtual CPU structure.
7774 * @param pHostCpu Pointer to the global CPU info struct.
7775 */
7776VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu, PHMGLOBALCPUINFO pHostCpu)
7777{
7778 AssertPtr(pVCpu);
7779 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
7780 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7781 RT_NOREF(pHostCpu);
7782
7783 LogFlowFunc(("pVCpu=%p\n", pVCpu));
7784 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7785 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7786
7787#ifdef VBOX_STRICT
7788 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
7789 RTCCUINTREG uHostCR4 = ASMGetCR4();
7790 if (!(uHostCR4 & X86_CR4_VMXE))
7791 {
7792 LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
7793 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7794 }
7795#endif
7796
7797 /*
7798 * Load the VCPU's VMCS as the current (and active) one.
7799 */
7800 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7801 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7802 if (RT_FAILURE(rc))
7803 return rc;
7804
7805 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7806 pVCpu->hm.s.fLeaveDone = false;
7807 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7808
7809 return VINF_SUCCESS;
7810}
7811
7812
7813/**
7814 * The thread-context callback (only on platforms which support it).
7815 *
7816 * @param enmEvent The thread-context event.
7817 * @param pVCpu The cross context virtual CPU structure.
7818 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7819 * @thread EMT(pVCpu)
7820 */
7821VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7822{
7823 NOREF(fGlobalInit);
7824
7825 switch (enmEvent)
7826 {
7827 case RTTHREADCTXEVENT_OUT:
7828 {
7829 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7830 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7831 VMCPU_ASSERT_EMT(pVCpu);
7832
7833 /* No longjmps (logger flushes, locks) in this fragile context. */
7834 VMMRZCallRing3Disable(pVCpu);
7835 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7836
7837 /*
7838 * Restore host-state (FPU, debug etc.)
7839 */
7840 if (!pVCpu->hm.s.fLeaveDone)
7841 {
7842 /*
7843 * Do -not- import the guest-state here as we might already be in the middle of importing
7844 * it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
7845 */
7846 hmR0VmxLeave(pVCpu, false /* fImportState */);
7847 pVCpu->hm.s.fLeaveDone = true;
7848 }
7849
7850 /* Leave HM context, takes care of local init (term). */
7851 int rc = HMR0LeaveCpu(pVCpu);
7852 AssertRC(rc); NOREF(rc);
7853
7854 /* Restore longjmp state. */
7855 VMMRZCallRing3Enable(pVCpu);
7856 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
7857 break;
7858 }
7859
7860 case RTTHREADCTXEVENT_IN:
7861 {
7862 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7863 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7864 VMCPU_ASSERT_EMT(pVCpu);
7865
7866 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7867 VMMRZCallRing3Disable(pVCpu);
7868 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7869
7870 /* Initialize the bare minimum state required for HM. This takes care of
7871 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7872 int rc = hmR0EnterCpu(pVCpu);
7873 AssertRC(rc);
7874 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7875 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7876
7877 /* Load the active VMCS as the current one. */
7878 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7879 {
7880 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7881 AssertRC(rc); NOREF(rc);
7882 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7883 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7884 }
7885 pVCpu->hm.s.fLeaveDone = false;
7886
7887 /* Restore longjmp state. */
7888 VMMRZCallRing3Enable(pVCpu);
7889 break;
7890 }
7891
7892 default:
7893 break;
7894 }
7895}
7896
7897
7898/**
7899 * Exports the host state into the VMCS host-state area.
7900 * Sets up the VM-exit MSR-load area.
7901 *
7902 * The CPU state will be loaded from these fields on every successful VM-exit.
7903 *
7904 * @returns VBox status code.
7905 * @param pVCpu The cross context virtual CPU structure.
7906 *
7907 * @remarks No-long-jump zone!!!
7908 */
7909static int hmR0VmxExportHostState(PVMCPU pVCpu)
7910{
7911 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7912
7913 int rc = VINF_SUCCESS;
7914 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
7915 {
7916 rc = hmR0VmxExportHostControlRegs();
7917 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7918
7919 rc = hmR0VmxExportHostSegmentRegs(pVCpu);
7920 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7921
7922 rc = hmR0VmxExportHostMsrs(pVCpu);
7923 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7924
7925 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
7926 }
7927 return rc;
7928}
7929
7930
7931/**
7932 * Saves the host state in the VMCS host-state.
7933 *
7934 * @returns VBox status code.
7935 * @param pVCpu The cross context virtual CPU structure.
7936 *
7937 * @remarks No-long-jump zone!!!
7938 */
7939VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
7940{
7941 AssertPtr(pVCpu);
7942 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7943
7944 /*
7945 * Export the host state here while entering HM context.
7946 * When thread-context hooks are used, we might get preempted and have to re-save the host
7947 * state but most of the time we won't be, so do it here before we disable interrupts.
7948 */
7949 return hmR0VmxExportHostState(pVCpu);
7950}
7951
7952
7953/**
7954 * Exports the guest state into the VMCS guest-state area.
7955 *
7956 * The will typically be done before VM-entry when the guest-CPU state and the
7957 * VMCS state may potentially be out of sync.
7958 *
7959 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
7960 * VM-entry controls.
7961 * Sets up the appropriate VMX non-root function to execute guest code based on
7962 * the guest CPU mode.
7963 *
7964 * @returns VBox strict status code.
7965 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
7966 * without unrestricted guest access and the VMMDev is not presently
7967 * mapped (e.g. EFI32).
7968 *
7969 * @param pVCpu The cross context virtual CPU structure.
7970 *
7971 * @remarks No-long-jump zone!!!
7972 */
7973static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu)
7974{
7975 AssertPtr(pVCpu);
7976 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7977
7978 LogFlowFunc(("pVCpu=%p\n", pVCpu));
7979
7980 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
7981
7982 /* Determine real-on-v86 mode. */
7983 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7984 if ( !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
7985 && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
7986 {
7987 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7988 }
7989
7990 /*
7991 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7992 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
7993 */
7994 int rc = hmR0VmxSelectVMRunHandler(pVCpu);
7995 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7996
7997 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7998 rc = hmR0VmxExportGuestEntryCtls(pVCpu);
7999 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8000
8001 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8002 rc = hmR0VmxExportGuestExitCtls(pVCpu);
8003 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8004
8005 rc = hmR0VmxExportGuestCR0(pVCpu);
8006 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8007
8008 VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu);
8009 if (rcStrict == VINF_SUCCESS)
8010 { /* likely */ }
8011 else
8012 {
8013 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8014 return rcStrict;
8015 }
8016
8017 rc = hmR0VmxExportGuestSegmentRegs(pVCpu);
8018 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8019
8020 /* This needs to be done after hmR0VmxExportGuestEntryCtls() and hmR0VmxExportGuestExitCtls() as it
8021 may alter controls if we determine we don't have to swap EFER after all. */
8022 rc = hmR0VmxExportGuestMsrs(pVCpu);
8023 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8024
8025 rc = hmR0VmxExportGuestApicTpr(pVCpu);
8026 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8027
8028 rc = hmR0VmxExportGuestXcptIntercepts(pVCpu);
8029 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8030
8031 /* Exporting RFLAGS here is fine, even though RFLAGS.TF might depend on guest debug state which is
8032 not exported here. It is re-evaluated and updated if necessary in hmR0VmxExportSharedState(). */
8033 rc = hmR0VmxExportGuestRip(pVCpu);
8034 rc |= hmR0VmxExportGuestRsp(pVCpu);
8035 rc |= hmR0VmxExportGuestRflags(pVCpu);
8036 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8037
8038 /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
8039 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
8040 | HM_CHANGED_GUEST_CR2
8041 | (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
8042 | HM_CHANGED_GUEST_X87
8043 | HM_CHANGED_GUEST_SSE_AVX
8044 | HM_CHANGED_GUEST_OTHER_XSAVE
8045 | HM_CHANGED_GUEST_XCRx
8046 | HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
8047 | HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
8048 | HM_CHANGED_GUEST_TSC_AUX
8049 | HM_CHANGED_GUEST_OTHER_MSRS
8050 | HM_CHANGED_GUEST_HWVIRT
8051 | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
8052
8053 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
8054 return rc;
8055}
8056
8057
8058/**
8059 * Exports the state shared between the host and guest into the VMCS.
8060 *
8061 * @param pVCpu The cross context virtual CPU structure.
8062 *
8063 * @remarks No-long-jump zone!!!
8064 */
8065static void hmR0VmxExportSharedState(PVMCPU pVCpu)
8066{
8067 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8068 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8069
8070 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
8071 {
8072 int rc = hmR0VmxExportSharedDebugState(pVCpu);
8073 AssertRC(rc);
8074 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
8075
8076 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8077 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
8078 {
8079 rc = hmR0VmxExportGuestRflags(pVCpu);
8080 AssertRC(rc);
8081 }
8082 }
8083
8084 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
8085 {
8086 hmR0VmxLazyLoadGuestMsrs(pVCpu);
8087 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
8088 }
8089
8090 AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
8091 ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8092}
8093
8094
8095/**
8096 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8097 *
8098 * @returns Strict VBox status code (i.e. informational status codes too).
8099 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8100 * without unrestricted guest access and the VMMDev is not presently
8101 * mapped (e.g. EFI32).
8102 *
8103 * @param pVCpu The cross context virtual CPU structure.
8104 *
8105 * @remarks No-long-jump zone!!!
8106 */
8107static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu)
8108{
8109 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8110 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8111 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8112
8113#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8114 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8115#endif
8116
8117 /*
8118 * For many exits it's only RIP that changes and hence try to export it first
8119 * without going through a lot of change flag checks.
8120 */
8121 VBOXSTRICTRC rcStrict;
8122 uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8123 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8124 if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
8125 {
8126 rcStrict = hmR0VmxExportGuestRip(pVCpu);
8127 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8128 { /* likely */}
8129 else
8130 AssertMsgFailedReturn(("hmR0VmxExportGuestRip failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8131 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
8132 }
8133 else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8134 {
8135 rcStrict = hmR0VmxExportGuestState(pVCpu);
8136 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8137 { /* likely */}
8138 else
8139 {
8140 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("hmR0VmxExportGuestState failed! rc=%Rrc\n",
8141 VBOXSTRICTRC_VAL(rcStrict)));
8142 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8143 return rcStrict;
8144 }
8145 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
8146 }
8147 else
8148 rcStrict = VINF_SUCCESS;
8149
8150#ifdef VBOX_STRICT
8151 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8152 fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8153 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8154 AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
8155 ("fCtxChanged=%#RX64\n", fCtxChanged));
8156#endif
8157 return rcStrict;
8158}
8159
8160
8161/**
8162 * Does the preparations before executing guest code in VT-x.
8163 *
8164 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8165 * recompiler/IEM. We must be cautious what we do here regarding committing
8166 * guest-state information into the VMCS assuming we assuredly execute the
8167 * guest in VT-x mode.
8168 *
8169 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8170 * the common-state (TRPM/forceflags), we must undo those changes so that the
8171 * recompiler/IEM can (and should) use them when it resumes guest execution.
8172 * Otherwise such operations must be done when we can no longer exit to ring-3.
8173 *
8174 * @returns Strict VBox status code (i.e. informational status codes too).
8175 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8176 * have been disabled.
8177 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8178 * double-fault into the guest.
8179 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8180 * dispatched directly.
8181 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8182 *
8183 * @param pVCpu The cross context virtual CPU structure.
8184 * @param pVmxTransient Pointer to the VMX transient structure.
8185 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8186 * us ignore some of the reasons for returning to
8187 * ring-3, and return VINF_EM_DBG_STEPPED if event
8188 * dispatching took place.
8189 */
8190static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
8191{
8192 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8193
8194#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8195 PGMRZDynMapFlushAutoSet(pVCpu);
8196#endif
8197
8198 /* Check force flag actions that might require us to go back to ring-3. */
8199 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
8200 if (rcStrict == VINF_SUCCESS)
8201 { /* FFs doesn't get set all the time. */ }
8202 else
8203 return rcStrict;
8204
8205 /*
8206 * Setup the virtualized-APIC accesses.
8207 *
8208 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8209 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8210 *
8211 * This is the reason we do it here and not in hmR0VmxExportGuestState().
8212 */
8213 PVM pVM = pVCpu->CTX_SUFF(pVM);
8214 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8215 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
8216 && PDMHasApic(pVM))
8217 {
8218 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8219 Assert(u64MsrApicBase);
8220 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8221
8222 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8223
8224 /* Unalias any existing mapping. */
8225 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8226 AssertRCReturn(rc, rc);
8227
8228 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8229 Log4Func(("Mapped HC APIC-access page at %#RGp\n", GCPhysApicBase));
8230 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8231 AssertRCReturn(rc, rc);
8232
8233 /* Update the per-VCPU cache of the APIC base MSR. */
8234 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8235 }
8236
8237 if (TRPMHasTrap(pVCpu))
8238 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8239 uint32_t fIntrState = hmR0VmxEvaluatePendingEvent(pVCpu);
8240
8241 /*
8242 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
8243 * needs to be done with longjmps or interrupts + preemption enabled. Event injection might
8244 * also result in triple-faulting the VM.
8245 */
8246 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, fIntrState, fStepping);
8247 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8248 { /* likely */ }
8249 else
8250 {
8251 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8252 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8253 return rcStrict;
8254 }
8255
8256 /*
8257 * A longjump might result in importing CR3 even for VM-exits that don't necessarily
8258 * import CR3 themselves. We will need to update them here, as even as late as the above
8259 * hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
8260 * the below force flags to be set.
8261 */
8262 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8263 {
8264 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
8265 int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8266 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
8267 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
8268 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8269 }
8270 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8271 {
8272 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8273 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8274 }
8275
8276 /*
8277 * No longjmps to ring-3 from this point on!!!
8278 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8279 * This also disables flushing of the R0-logger instance (if any).
8280 */
8281 VMMRZCallRing3Disable(pVCpu);
8282
8283 /*
8284 * Export the guest state bits.
8285 *
8286 * We cannot perform longjmps while loading the guest state because we do not preserve the
8287 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8288 * CPU migration.
8289 *
8290 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8291 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8292 * Hence, loading of the guest state needs to be done -after- injection of events.
8293 */
8294 rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu);
8295 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8296 { /* likely */ }
8297 else
8298 {
8299 VMMRZCallRing3Enable(pVCpu);
8300 return rcStrict;
8301 }
8302
8303 /*
8304 * We disable interrupts so that we don't miss any interrupts that would flag preemption
8305 * (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
8306 * preemption disabled for a while. Since this is purly to aid the
8307 * RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
8308 * disable interrupt on NT.
8309 *
8310 * We need to check for force-flags that could've possible been altered since we last
8311 * checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
8312 * see @bugref{6398}).
8313 *
8314 * We also check a couple of other force-flags as a last opportunity to get the EMT back
8315 * to ring-3 before executing guest code.
8316 */
8317 pVmxTransient->fEFlags = ASMIntDisableFlags();
8318
8319 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8320 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8321 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8322 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8323 {
8324 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8325 {
8326 pVCpu->hm.s.Event.fPending = false;
8327
8328 /*
8329 * We've injected any pending events. This is really the point of no return (to ring-3).
8330 *
8331 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
8332 * returns from this function, so don't enable them here.
8333 */
8334 return VINF_SUCCESS;
8335 }
8336
8337 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
8338 rcStrict = VINF_EM_RAW_INTERRUPT;
8339 }
8340 else
8341 {
8342 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8343 rcStrict = VINF_EM_RAW_TO_R3;
8344 }
8345
8346 ASMSetFlags(pVmxTransient->fEFlags);
8347 VMMRZCallRing3Enable(pVCpu);
8348
8349 return rcStrict;
8350}
8351
8352
8353/**
8354 * Prepares to run guest code in VT-x and we've committed to doing so. This
8355 * means there is no backing out to ring-3 or anywhere else at this
8356 * point.
8357 *
8358 * @param pVCpu The cross context virtual CPU structure.
8359 * @param pVmxTransient Pointer to the VMX transient structure.
8360 *
8361 * @remarks Called with preemption disabled.
8362 * @remarks No-long-jump zone!!!
8363 */
8364static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
8365{
8366 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8367 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8368 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8369
8370 /*
8371 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8372 */
8373 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8374 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8375
8376 PVM pVM = pVCpu->CTX_SUFF(pVM);
8377 if (!CPUMIsGuestFPUStateActive(pVCpu))
8378 {
8379 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8380 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8381 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT;
8382 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8383 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
8384 }
8385
8386 /*
8387 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8388 */
8389 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8390 && pVCpu->hm.s.vmx.cMsrs > 0)
8391 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8392
8393 /*
8394 * Re-save the host state bits as we may've been preempted (only happens when
8395 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8396 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8397 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8398 * See @bugref{8432}.
8399 */
8400 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8401 {
8402 int rc = hmR0VmxExportHostState(pVCpu);
8403 AssertRC(rc);
8404 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
8405 }
8406 Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
8407
8408 /*
8409 * Export the state shared between host and guest (FPU, debug, lazy MSRs).
8410 */
8411 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
8412 hmR0VmxExportSharedState(pVCpu);
8413 AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8414
8415 /* Store status of the shared guest-host state at the time of VM-entry. */
8416#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8417 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
8418 {
8419 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8420 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8421 }
8422 else
8423#endif
8424 {
8425 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8426 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8427 }
8428
8429 /*
8430 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8431 */
8432 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
8433 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8434
8435 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
8436 RTCPUID idCurrentCpu = pCpu->idCpu;
8437 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8438 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8439 {
8440 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8441 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8442 }
8443
8444 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8445 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8446 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8447 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8448
8449 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8450
8451 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8452 to start executing. */
8453
8454 /*
8455 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8456 */
8457 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
8458 {
8459 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
8460 {
8461 bool fMsrUpdated;
8462 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
8463 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8464 &fMsrUpdated);
8465 AssertRC(rc2);
8466 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8467 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8468 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8469 }
8470 else
8471 {
8472 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8473 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8474 }
8475 }
8476
8477 if (pVM->cpum.ro.GuestFeatures.fIbrs)
8478 {
8479 bool fMsrUpdated;
8480 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_OTHER_MSRS);
8481 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
8482 &fMsrUpdated);
8483 AssertRC(rc2);
8484 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8485 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8486 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8487 }
8488
8489#ifdef VBOX_STRICT
8490 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8491 hmR0VmxCheckHostEferMsr(pVCpu);
8492 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8493#endif
8494#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8495 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
8496 {
8497 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
8498 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8499 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8500 }
8501#endif
8502}
8503
8504
8505/**
8506 * Performs some essential restoration of state after running guest code in
8507 * VT-x.
8508 *
8509 * @param pVCpu The cross context virtual CPU structure.
8510 * @param pVmxTransient Pointer to the VMX transient structure.
8511 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8512 *
8513 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8514 *
8515 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8516 * unconditionally when it is safe to do so.
8517 */
8518static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8519{
8520 uint64_t const uHostTsc = ASMReadTSC();
8521 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8522
8523 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8524 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8525 pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8526 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8527 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8528 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8529
8530 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
8531 TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TscOffset);
8532
8533 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
8534 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8535 Assert(!ASMIntAreEnabled());
8536 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8537
8538#if HC_ARCH_BITS == 64
8539 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8540#endif
8541#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8542 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8543 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8544 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8545#else
8546 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8547#endif
8548#ifdef VBOX_STRICT
8549 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8550#endif
8551 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8552
8553 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8554 uint32_t uExitReason;
8555 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8556 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8557 AssertRC(rc);
8558 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8559 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INT_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8560
8561 if (rcVMRun == VINF_SUCCESS)
8562 {
8563 /*
8564 * Update the VM-exit history array here even if the VM-entry failed due to:
8565 * - Invalid guest state.
8566 * - MSR loading.
8567 * - Machine-check event.
8568 *
8569 * In any of the above cases we will still have a "valid" VM-exit reason
8570 * despite @a fVMEntryFailed being false.
8571 *
8572 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8573 *
8574 * Note! We don't have CS or RIP at this point. Will probably address that later
8575 * by amending the history entry added here.
8576 */
8577 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
8578 UINT64_MAX, uHostTsc);
8579
8580 if (!pVmxTransient->fVMEntryFailed)
8581 {
8582 VMMRZCallRing3Enable(pVCpu);
8583
8584 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8585 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8586
8587#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8588 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
8589 AssertRC(rc);
8590#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8591 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_RFLAGS);
8592 AssertRC(rc);
8593#else
8594 /*
8595 * Import the guest-interruptibility state always as we need it while evaluating
8596 * injecting events on re-entry.
8597 *
8598 * We don't import CR0 (when Unrestricted guest execution is unavailable) despite
8599 * checking for real-mode while exporting the state because all bits that cause
8600 * mode changes wrt CR0 are intercepted.
8601 */
8602 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
8603 AssertRC(rc);
8604#endif
8605
8606 /*
8607 * Sync the TPR shadow with our APIC state.
8608 */
8609 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
8610 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8611 {
8612 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8613 AssertRC(rc);
8614 ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8615 }
8616
8617 return;
8618 }
8619 }
8620 else
8621 Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
8622
8623 VMMRZCallRing3Enable(pVCpu);
8624}
8625
8626
8627/**
8628 * Runs the guest code using VT-x the normal way.
8629 *
8630 * @returns VBox status code.
8631 * @param pVCpu The cross context virtual CPU structure.
8632 *
8633 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8634 */
8635static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu)
8636{
8637 VMXTRANSIENT VmxTransient;
8638 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8639 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8640 uint32_t cLoops = 0;
8641
8642 for (;; cLoops++)
8643 {
8644 Assert(!HMR0SuspendPending());
8645 HMVMX_ASSERT_CPU_SAFE(pVCpu);
8646
8647 /* Preparatory work for running guest code, this may force us to return
8648 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8649 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8650 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
8651 if (rcStrict != VINF_SUCCESS)
8652 break;
8653
8654 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
8655 int rcRun = hmR0VmxRunGuest(pVCpu);
8656
8657 /* Restore any residual host-state and save any bits shared between host
8658 and guest into the guest-CPU state. Re-enables interrupts! */
8659 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
8660
8661 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8662 if (RT_SUCCESS(rcRun))
8663 { /* very likely */ }
8664 else
8665 {
8666 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
8667 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
8668 return rcRun;
8669 }
8670
8671 /* Profile the VM-exit. */
8672 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8673 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8674 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8675 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
8676 HMVMX_START_EXIT_DISPATCH_PROF();
8677
8678 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
8679
8680 /* Handle the VM-exit. */
8681#ifdef HMVMX_USE_FUNCTION_TABLE
8682 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
8683#else
8684 rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient, VmxTransient.uExitReason);
8685#endif
8686 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
8687 if (rcStrict == VINF_SUCCESS)
8688 {
8689 if (cLoops <= pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
8690 continue; /* likely */
8691 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8692 rcStrict = VINF_EM_RAW_INTERRUPT;
8693 }
8694 break;
8695 }
8696
8697 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8698 return rcStrict;
8699}
8700
8701
8702
8703/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8704 * probes.
8705 *
8706 * The following few functions and associated structure contains the bloat
8707 * necessary for providing detailed debug events and dtrace probes as well as
8708 * reliable host side single stepping. This works on the principle of
8709 * "subclassing" the normal execution loop and workers. We replace the loop
8710 * method completely and override selected helpers to add necessary adjustments
8711 * to their core operation.
8712 *
8713 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8714 * any performance for debug and analysis features.
8715 *
8716 * @{
8717 */
8718
8719/**
8720 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
8721 * the debug run loop.
8722 */
8723typedef struct VMXRUNDBGSTATE
8724{
8725 /** The RIP we started executing at. This is for detecting that we stepped. */
8726 uint64_t uRipStart;
8727 /** The CS we started executing with. */
8728 uint16_t uCsStart;
8729
8730 /** Whether we've actually modified the 1st execution control field. */
8731 bool fModifiedProcCtls : 1;
8732 /** Whether we've actually modified the 2nd execution control field. */
8733 bool fModifiedProcCtls2 : 1;
8734 /** Whether we've actually modified the exception bitmap. */
8735 bool fModifiedXcptBitmap : 1;
8736
8737 /** We desire the modified the CR0 mask to be cleared. */
8738 bool fClearCr0Mask : 1;
8739 /** We desire the modified the CR4 mask to be cleared. */
8740 bool fClearCr4Mask : 1;
8741 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
8742 uint32_t fCpe1Extra;
8743 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
8744 uint32_t fCpe1Unwanted;
8745 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
8746 uint32_t fCpe2Extra;
8747 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
8748 uint32_t bmXcptExtra;
8749 /** The sequence number of the Dtrace provider settings the state was
8750 * configured against. */
8751 uint32_t uDtraceSettingsSeqNo;
8752 /** VM-exits to check (one bit per VM-exit). */
8753 uint32_t bmExitsToCheck[3];
8754
8755 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
8756 uint32_t fProcCtlsInitial;
8757 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
8758 uint32_t fProcCtls2Initial;
8759 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
8760 uint32_t bmXcptInitial;
8761} VMXRUNDBGSTATE;
8762AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
8763typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
8764
8765
8766/**
8767 * Initializes the VMXRUNDBGSTATE structure.
8768 *
8769 * @param pVCpu The cross context virtual CPU structure of the
8770 * calling EMT.
8771 * @param pDbgState The structure to initialize.
8772 */
8773static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
8774{
8775 pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
8776 pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
8777
8778 pDbgState->fModifiedProcCtls = false;
8779 pDbgState->fModifiedProcCtls2 = false;
8780 pDbgState->fModifiedXcptBitmap = false;
8781 pDbgState->fClearCr0Mask = false;
8782 pDbgState->fClearCr4Mask = false;
8783 pDbgState->fCpe1Extra = 0;
8784 pDbgState->fCpe1Unwanted = 0;
8785 pDbgState->fCpe2Extra = 0;
8786 pDbgState->bmXcptExtra = 0;
8787 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
8788 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
8789 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
8790}
8791
8792
8793/**
8794 * Updates the VMSC fields with changes requested by @a pDbgState.
8795 *
8796 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
8797 * immediately before executing guest code, i.e. when interrupts are disabled.
8798 * We don't check status codes here as we cannot easily assert or return in the
8799 * latter case.
8800 *
8801 * @param pVCpu The cross context virtual CPU structure.
8802 * @param pDbgState The debug state.
8803 */
8804static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
8805{
8806 /*
8807 * Ensure desired flags in VMCS control fields are set.
8808 * (Ignoring write failure here, as we're committed and it's just debug extras.)
8809 *
8810 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
8811 * there should be no stale data in pCtx at this point.
8812 */
8813 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
8814 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
8815 {
8816 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
8817 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
8818 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8819 Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
8820 pDbgState->fModifiedProcCtls = true;
8821 }
8822
8823 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
8824 {
8825 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
8826 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
8827 Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
8828 pDbgState->fModifiedProcCtls2 = true;
8829 }
8830
8831 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
8832 {
8833 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
8834 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8835 Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
8836 pDbgState->fModifiedXcptBitmap = true;
8837 }
8838
8839 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32Cr0Mask != 0)
8840 {
8841 pVCpu->hm.s.vmx.u32Cr0Mask = 0;
8842 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
8843 Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
8844 }
8845
8846 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32Cr4Mask != 0)
8847 {
8848 pVCpu->hm.s.vmx.u32Cr4Mask = 0;
8849 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
8850 Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
8851 }
8852}
8853
8854
8855/**
8856 * Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
8857 * re-entry next time around.
8858 *
8859 * @returns Strict VBox status code (i.e. informational status codes too).
8860 * @param pVCpu The cross context virtual CPU structure.
8861 * @param pDbgState The debug state.
8862 * @param rcStrict The return code from executing the guest using single
8863 * stepping.
8864 */
8865static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
8866{
8867 /*
8868 * Restore VM-exit control settings as we may not reenter this function the
8869 * next time around.
8870 */
8871 /* We reload the initial value, trigger what we can of recalculations the
8872 next time around. From the looks of things, that's all that's required atm. */
8873 if (pDbgState->fModifiedProcCtls)
8874 {
8875 if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
8876 pDbgState->fProcCtlsInitial |= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
8877 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
8878 AssertRCReturn(rc2, rc2);
8879 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
8880 }
8881
8882 /* We're currently the only ones messing with this one, so just restore the
8883 cached value and reload the field. */
8884 if ( pDbgState->fModifiedProcCtls2
8885 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
8886 {
8887 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
8888 AssertRCReturn(rc2, rc2);
8889 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
8890 }
8891
8892 /* If we've modified the exception bitmap, we restore it and trigger
8893 reloading and partial recalculation the next time around. */
8894 if (pDbgState->fModifiedXcptBitmap)
8895 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
8896
8897 return rcStrict;
8898}
8899
8900
8901/**
8902 * Configures VM-exit controls for current DBGF and DTrace settings.
8903 *
8904 * This updates @a pDbgState and the VMCS execution control fields to reflect
8905 * the necessary VM-exits demanded by DBGF and DTrace.
8906 *
8907 * @param pVCpu The cross context virtual CPU structure.
8908 * @param pDbgState The debug state.
8909 * @param pVmxTransient Pointer to the VMX transient structure. May update
8910 * fUpdateTscOffsettingAndPreemptTimer.
8911 */
8912static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
8913{
8914 /*
8915 * Take down the dtrace serial number so we can spot changes.
8916 */
8917 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
8918 ASMCompilerBarrier();
8919
8920 /*
8921 * We'll rebuild most of the middle block of data members (holding the
8922 * current settings) as we go along here, so start by clearing it all.
8923 */
8924 pDbgState->bmXcptExtra = 0;
8925 pDbgState->fCpe1Extra = 0;
8926 pDbgState->fCpe1Unwanted = 0;
8927 pDbgState->fCpe2Extra = 0;
8928 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
8929 pDbgState->bmExitsToCheck[i] = 0;
8930
8931 /*
8932 * Software interrupts (INT XXh) - no idea how to trigger these...
8933 */
8934 PVM pVM = pVCpu->CTX_SUFF(pVM);
8935 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
8936 || VBOXVMM_INT_SOFTWARE_ENABLED())
8937 {
8938 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
8939 }
8940
8941 /*
8942 * INT3 breakpoints - triggered by #BP exceptions.
8943 */
8944 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
8945 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
8946
8947 /*
8948 * Exception bitmap and XCPT events+probes.
8949 */
8950 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
8951 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
8952 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
8953
8954 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
8955 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
8956 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
8957 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
8958 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
8959 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
8960 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
8961 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
8962 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
8963 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
8964 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
8965 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
8966 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
8967 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
8968 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
8969 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
8970 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
8971 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
8972
8973 if (pDbgState->bmXcptExtra)
8974 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
8975
8976 /*
8977 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
8978 *
8979 * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
8980 * So, when adding/changing/removing please don't forget to update it.
8981 *
8982 * Some of the macros are picking up local variables to save horizontal space,
8983 * (being able to see it in a table is the lesser evil here).
8984 */
8985#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
8986 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
8987 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
8988#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
8989 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
8990 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
8991 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
8992 } else do { } while (0)
8993#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
8994 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
8995 { \
8996 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
8997 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
8998 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
8999 } else do { } while (0)
9000#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9001 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9002 { \
9003 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9004 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9005 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9006 } else do { } while (0)
9007#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9008 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9009 { \
9010 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9011 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9012 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9013 } else do { } while (0)
9014
9015 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9016 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9017 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9018 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9019 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9020
9021 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9022 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9023 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9024 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9025 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
9026 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9027 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9028 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9029 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
9030 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9031 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
9032 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9033 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
9034 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9035 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9036 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9037 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9038 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9039 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9040 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9041 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9042 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9043 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9044 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9045 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9046 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9047 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9048 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9049 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9050 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9051 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9052 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9053 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9054 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9055 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9056 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9057
9058 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9059 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9060 {
9061 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_APIC_TPR);
9062 AssertRC(rc);
9063
9064#if 0 /** @todo fix me */
9065 pDbgState->fClearCr0Mask = true;
9066 pDbgState->fClearCr4Mask = true;
9067#endif
9068 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9069 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_CR3_STORE_EXIT | VMX_PROC_CTLS_CR8_STORE_EXIT;
9070 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9071 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_CR3_LOAD_EXIT | VMX_PROC_CTLS_CR8_LOAD_EXIT;
9072 pDbgState->fCpe1Unwanted |= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
9073 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9074 require clearing here and in the loop if we start using it. */
9075 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9076 }
9077 else
9078 {
9079 if (pDbgState->fClearCr0Mask)
9080 {
9081 pDbgState->fClearCr0Mask = false;
9082 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
9083 }
9084 if (pDbgState->fClearCr4Mask)
9085 {
9086 pDbgState->fClearCr4Mask = false;
9087 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
9088 }
9089 }
9090 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9091 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9092
9093 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9094 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9095 {
9096 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9097 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9098 }
9099 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9100 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9101
9102 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
9103 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9104 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
9105 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9106 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
9107 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9108 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
9109 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9110#if 0 /** @todo too slow, fix handler. */
9111 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
9112#endif
9113 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9114
9115 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9116 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9117 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9118 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9119 {
9120 pDbgState->fCpe2Extra |= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
9121 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9122 }
9123 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9124 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9125 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9126 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9127
9128 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9129 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9130 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9131 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9132 {
9133 pDbgState->fCpe2Extra |= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
9134 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9135 }
9136 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9137 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9138 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9139 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9140
9141 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9142 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9143 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
9144 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9145 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9146 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9147 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
9148 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9149 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9150 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9151 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
9152 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9153 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
9154 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9155 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9156 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9157 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
9158 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9159 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9160 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9161 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9162 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9163
9164#undef IS_EITHER_ENABLED
9165#undef SET_ONLY_XBM_IF_EITHER_EN
9166#undef SET_CPE1_XBM_IF_EITHER_EN
9167#undef SET_CPEU_XBM_IF_EITHER_EN
9168#undef SET_CPE2_XBM_IF_EITHER_EN
9169
9170 /*
9171 * Sanitize the control stuff.
9172 */
9173 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
9174 if (pDbgState->fCpe2Extra)
9175 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
9176 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
9177 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0;
9178 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
9179 {
9180 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9181 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9182 }
9183
9184 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9185 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9186 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9187 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9188}
9189
9190
9191/**
9192 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9193 * appropriate.
9194 *
9195 * The caller has checked the VM-exit against the
9196 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9197 * already, so we don't have to do that either.
9198 *
9199 * @returns Strict VBox status code (i.e. informational status codes too).
9200 * @param pVCpu The cross context virtual CPU structure.
9201 * @param pVmxTransient Pointer to the VMX-transient structure.
9202 * @param uExitReason The VM-exit reason.
9203 *
9204 * @remarks The name of this function is displayed by dtrace, so keep it short
9205 * and to the point. No longer than 33 chars long, please.
9206 */
9207static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9208{
9209 /*
9210 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9211 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9212 *
9213 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9214 * does. Must add/change/remove both places. Same ordering, please.
9215 *
9216 * Added/removed events must also be reflected in the next section
9217 * where we dispatch dtrace events.
9218 */
9219 bool fDtrace1 = false;
9220 bool fDtrace2 = false;
9221 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9222 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9223 uint32_t uEventArg = 0;
9224#define SET_EXIT(a_EventSubName) \
9225 do { \
9226 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9227 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9228 } while (0)
9229#define SET_BOTH(a_EventSubName) \
9230 do { \
9231 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9232 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9233 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9234 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9235 } while (0)
9236 switch (uExitReason)
9237 {
9238 case VMX_EXIT_MTF:
9239 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9240
9241 case VMX_EXIT_XCPT_OR_NMI:
9242 {
9243 uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9244 switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
9245 {
9246 case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
9247 case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
9248 case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
9249 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9250 {
9251 if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
9252 {
9253 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9254 uEventArg = pVmxTransient->uExitIntErrorCode;
9255 }
9256 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9257 switch (enmEvent1)
9258 {
9259 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9260 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9261 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9262 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9263 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9264 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9265 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9266 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9267 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9268 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9269 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9270 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9271 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9272 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9273 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9274 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9275 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9276 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9277 default: break;
9278 }
9279 }
9280 else
9281 AssertFailed();
9282 break;
9283
9284 case VMX_EXIT_INT_INFO_TYPE_SW_INT:
9285 uEventArg = idxVector;
9286 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9287 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9288 break;
9289 }
9290 break;
9291 }
9292
9293 case VMX_EXIT_TRIPLE_FAULT:
9294 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9295 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9296 break;
9297 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9298 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9299 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9300 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9301 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9302
9303 /* Instruction specific VM-exits: */
9304 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9305 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9306 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9307 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9308 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9309 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9310 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9311 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9312 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9313 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9314 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9315 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9316 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9317 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9318 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9319 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9320 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9321 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9322 case VMX_EXIT_MOV_CRX:
9323 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9324 if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
9325 SET_BOTH(CRX_READ);
9326 else
9327 SET_BOTH(CRX_WRITE);
9328 uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQualification);
9329 break;
9330 case VMX_EXIT_MOV_DRX:
9331 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9332 if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification)
9333 == VMX_EXIT_QUAL_DRX_DIRECTION_READ)
9334 SET_BOTH(DRX_READ);
9335 else
9336 SET_BOTH(DRX_WRITE);
9337 uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification);
9338 break;
9339 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9340 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9341 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9342 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9343 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9344 case VMX_EXIT_XDTR_ACCESS:
9345 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9346 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
9347 {
9348 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9349 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9350 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9351 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9352 }
9353 break;
9354
9355 case VMX_EXIT_TR_ACCESS:
9356 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9357 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
9358 {
9359 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9360 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9361 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9362 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9363 }
9364 break;
9365
9366 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9367 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9368 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9369 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9370 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9371 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9372 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9373 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9374 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9375 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9376 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9377
9378 /* Events that aren't relevant at this point. */
9379 case VMX_EXIT_EXT_INT:
9380 case VMX_EXIT_INT_WINDOW:
9381 case VMX_EXIT_NMI_WINDOW:
9382 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9383 case VMX_EXIT_PREEMPT_TIMER:
9384 case VMX_EXIT_IO_INSTR:
9385 break;
9386
9387 /* Errors and unexpected events. */
9388 case VMX_EXIT_INIT_SIGNAL:
9389 case VMX_EXIT_SIPI:
9390 case VMX_EXIT_IO_SMI:
9391 case VMX_EXIT_SMI:
9392 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9393 case VMX_EXIT_ERR_MSR_LOAD:
9394 case VMX_EXIT_ERR_MACHINE_CHECK:
9395 break;
9396
9397 default:
9398 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9399 break;
9400 }
9401#undef SET_BOTH
9402#undef SET_EXIT
9403
9404 /*
9405 * Dtrace tracepoints go first. We do them here at once so we don't
9406 * have to copy the guest state saving and stuff a few dozen times.
9407 * Down side is that we've got to repeat the switch, though this time
9408 * we use enmEvent since the probes are a subset of what DBGF does.
9409 */
9410 if (fDtrace1 || fDtrace2)
9411 {
9412 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9413 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9414 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9415 switch (enmEvent1)
9416 {
9417 /** @todo consider which extra parameters would be helpful for each probe. */
9418 case DBGFEVENT_END: break;
9419 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
9420 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
9421 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
9422 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
9423 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
9424 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
9425 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
9426 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
9427 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
9428 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
9429 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
9430 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
9431 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
9432 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
9433 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
9434 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
9435 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
9436 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
9437 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9438 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9439 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
9440 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
9441 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
9442 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
9443 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
9444 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
9445 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
9446 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9447 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9448 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9449 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9450 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9451 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
9452 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9453 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
9454 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
9455 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
9456 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
9457 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
9458 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
9459 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
9460 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
9461 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
9462 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
9463 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
9464 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
9465 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
9466 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
9467 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
9468 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
9469 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
9470 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
9471 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
9472 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
9473 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
9474 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
9475 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
9476 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
9477 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
9478 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
9479 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
9480 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
9481 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
9482 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
9483 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
9484 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
9485 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9486 }
9487 switch (enmEvent2)
9488 {
9489 /** @todo consider which extra parameters would be helpful for each probe. */
9490 case DBGFEVENT_END: break;
9491 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
9492 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9493 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
9494 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
9495 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
9496 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
9497 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
9498 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
9499 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
9500 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9501 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9502 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9503 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9504 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9505 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
9506 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9507 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
9508 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
9509 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
9510 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
9511 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
9512 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
9513 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
9514 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
9515 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
9516 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
9517 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
9518 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
9519 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
9520 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
9521 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
9522 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
9523 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
9524 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
9525 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
9526 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
9527 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
9528 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
9529 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
9530 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
9531 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
9532 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
9533 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
9534 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
9535 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
9536 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
9537 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
9538 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
9539 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
9540 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
9541 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
9542 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
9543 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9544 }
9545 }
9546
9547 /*
9548 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9549 * the DBGF call will do a full check).
9550 *
9551 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9552 * Note! If we have to events, we prioritize the first, i.e. the instruction
9553 * one, in order to avoid event nesting.
9554 */
9555 PVM pVM = pVCpu->CTX_SUFF(pVM);
9556 if ( enmEvent1 != DBGFEVENT_END
9557 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9558 {
9559 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
9560 if (rcStrict != VINF_SUCCESS)
9561 return rcStrict;
9562 }
9563 else if ( enmEvent2 != DBGFEVENT_END
9564 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9565 {
9566 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
9567 if (rcStrict != VINF_SUCCESS)
9568 return rcStrict;
9569 }
9570
9571 return VINF_SUCCESS;
9572}
9573
9574
9575/**
9576 * Single-stepping VM-exit filtering.
9577 *
9578 * This is preprocessing the VM-exits and deciding whether we've gotten far
9579 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9580 * handling is performed.
9581 *
9582 * @returns Strict VBox status code (i.e. informational status codes too).
9583 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9584 * @param pVmxTransient Pointer to the VMX-transient structure.
9585 * @param pDbgState The debug state.
9586 */
9587DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
9588{
9589 /*
9590 * Expensive (saves context) generic dtrace VM-exit probe.
9591 */
9592 uint32_t const uExitReason = pVmxTransient->uExitReason;
9593 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9594 { /* more likely */ }
9595 else
9596 {
9597 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9598 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9599 AssertRC(rc);
9600 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9601 }
9602
9603 /*
9604 * Check for host NMI, just to get that out of the way.
9605 */
9606 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9607 { /* normally likely */ }
9608 else
9609 {
9610 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9611 AssertRCReturn(rc2, rc2);
9612 uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
9613 if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
9614 return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
9615 }
9616
9617 /*
9618 * Check for single stepping event if we're stepping.
9619 */
9620 if (pVCpu->hm.s.fSingleInstruction)
9621 {
9622 switch (uExitReason)
9623 {
9624 case VMX_EXIT_MTF:
9625 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9626
9627 /* Various events: */
9628 case VMX_EXIT_XCPT_OR_NMI:
9629 case VMX_EXIT_EXT_INT:
9630 case VMX_EXIT_TRIPLE_FAULT:
9631 case VMX_EXIT_INT_WINDOW:
9632 case VMX_EXIT_NMI_WINDOW:
9633 case VMX_EXIT_TASK_SWITCH:
9634 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9635 case VMX_EXIT_APIC_ACCESS:
9636 case VMX_EXIT_EPT_VIOLATION:
9637 case VMX_EXIT_EPT_MISCONFIG:
9638 case VMX_EXIT_PREEMPT_TIMER:
9639
9640 /* Instruction specific VM-exits: */
9641 case VMX_EXIT_CPUID:
9642 case VMX_EXIT_GETSEC:
9643 case VMX_EXIT_HLT:
9644 case VMX_EXIT_INVD:
9645 case VMX_EXIT_INVLPG:
9646 case VMX_EXIT_RDPMC:
9647 case VMX_EXIT_RDTSC:
9648 case VMX_EXIT_RSM:
9649 case VMX_EXIT_VMCALL:
9650 case VMX_EXIT_VMCLEAR:
9651 case VMX_EXIT_VMLAUNCH:
9652 case VMX_EXIT_VMPTRLD:
9653 case VMX_EXIT_VMPTRST:
9654 case VMX_EXIT_VMREAD:
9655 case VMX_EXIT_VMRESUME:
9656 case VMX_EXIT_VMWRITE:
9657 case VMX_EXIT_VMXOFF:
9658 case VMX_EXIT_VMXON:
9659 case VMX_EXIT_MOV_CRX:
9660 case VMX_EXIT_MOV_DRX:
9661 case VMX_EXIT_IO_INSTR:
9662 case VMX_EXIT_RDMSR:
9663 case VMX_EXIT_WRMSR:
9664 case VMX_EXIT_MWAIT:
9665 case VMX_EXIT_MONITOR:
9666 case VMX_EXIT_PAUSE:
9667 case VMX_EXIT_XDTR_ACCESS:
9668 case VMX_EXIT_TR_ACCESS:
9669 case VMX_EXIT_INVEPT:
9670 case VMX_EXIT_RDTSCP:
9671 case VMX_EXIT_INVVPID:
9672 case VMX_EXIT_WBINVD:
9673 case VMX_EXIT_XSETBV:
9674 case VMX_EXIT_RDRAND:
9675 case VMX_EXIT_INVPCID:
9676 case VMX_EXIT_VMFUNC:
9677 case VMX_EXIT_RDSEED:
9678 case VMX_EXIT_XSAVES:
9679 case VMX_EXIT_XRSTORS:
9680 {
9681 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
9682 AssertRCReturn(rc, rc);
9683 if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
9684 || pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
9685 return VINF_EM_DBG_STEPPED;
9686 break;
9687 }
9688
9689 /* Errors and unexpected events: */
9690 case VMX_EXIT_INIT_SIGNAL:
9691 case VMX_EXIT_SIPI:
9692 case VMX_EXIT_IO_SMI:
9693 case VMX_EXIT_SMI:
9694 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9695 case VMX_EXIT_ERR_MSR_LOAD:
9696 case VMX_EXIT_ERR_MACHINE_CHECK:
9697 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9698 break;
9699
9700 default:
9701 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9702 break;
9703 }
9704 }
9705
9706 /*
9707 * Check for debugger event breakpoints and dtrace probes.
9708 */
9709 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9710 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9711 {
9712 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
9713 if (rcStrict != VINF_SUCCESS)
9714 return rcStrict;
9715 }
9716
9717 /*
9718 * Normal processing.
9719 */
9720#ifdef HMVMX_USE_FUNCTION_TABLE
9721 return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
9722#else
9723 return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
9724#endif
9725}
9726
9727
9728/**
9729 * Single steps guest code using VT-x.
9730 *
9731 * @returns Strict VBox status code (i.e. informational status codes too).
9732 * @param pVCpu The cross context virtual CPU structure.
9733 *
9734 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
9735 */
9736static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu)
9737{
9738 VMXTRANSIENT VmxTransient;
9739 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9740
9741 /* Set HMCPU indicators. */
9742 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
9743 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
9744 pVCpu->hm.s.fDebugWantRdTscExit = false;
9745 pVCpu->hm.s.fUsingDebugLoop = true;
9746
9747 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
9748 VMXRUNDBGSTATE DbgState;
9749 hmR0VmxRunDebugStateInit(pVCpu, &DbgState);
9750 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9751
9752 /*
9753 * The loop.
9754 */
9755 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9756 for (uint32_t cLoops = 0; ; cLoops++)
9757 {
9758 Assert(!HMR0SuspendPending());
9759 HMVMX_ASSERT_CPU_SAFE(pVCpu);
9760 bool fStepping = pVCpu->hm.s.fSingleInstruction;
9761
9762 /*
9763 * Preparatory work for running guest code, this may force us to return
9764 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
9765 */
9766 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9767 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
9768 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
9769 if (rcStrict != VINF_SUCCESS)
9770 break;
9771
9772 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
9773 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
9774
9775 /*
9776 * Now we can run the guest code.
9777 */
9778 int rcRun = hmR0VmxRunGuest(pVCpu);
9779
9780 /*
9781 * Restore any residual host-state and save any bits shared between host
9782 * and guest into the guest-CPU state. Re-enables interrupts!
9783 */
9784 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
9785
9786 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9787 if (RT_SUCCESS(rcRun))
9788 { /* very likely */ }
9789 else
9790 {
9791 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
9792 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
9793 return rcRun;
9794 }
9795
9796 /* Profile the VM-exit. */
9797 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9798 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9799 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9800 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
9801 HMVMX_START_EXIT_DISPATCH_PROF();
9802
9803 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
9804
9805 /*
9806 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
9807 */
9808 rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
9809 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
9810 if (rcStrict != VINF_SUCCESS)
9811 break;
9812 if (cLoops > pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
9813 {
9814 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9815 rcStrict = VINF_EM_RAW_INTERRUPT;
9816 break;
9817 }
9818
9819 /*
9820 * Stepping: Did the RIP change, if so, consider it a single step.
9821 * Otherwise, make sure one of the TFs gets set.
9822 */
9823 if (fStepping)
9824 {
9825 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
9826 AssertRC(rc);
9827 if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
9828 || pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
9829 {
9830 rcStrict = VINF_EM_DBG_STEPPED;
9831 break;
9832 }
9833 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
9834 }
9835
9836 /*
9837 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
9838 */
9839 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
9840 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9841 }
9842
9843 /*
9844 * Clear the X86_EFL_TF if necessary.
9845 */
9846 if (pVCpu->hm.s.fClearTrapFlag)
9847 {
9848 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
9849 AssertRC(rc);
9850 pVCpu->hm.s.fClearTrapFlag = false;
9851 pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
9852 }
9853 /** @todo there seems to be issues with the resume flag when the monitor trap
9854 * flag is pending without being used. Seen early in bios init when
9855 * accessing APIC page in protected mode. */
9856
9857 /*
9858 * Restore VM-exit control settings as we may not reenter this function the
9859 * next time around.
9860 */
9861 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
9862
9863 /* Restore HMCPU indicators. */
9864 pVCpu->hm.s.fUsingDebugLoop = false;
9865 pVCpu->hm.s.fDebugWantRdTscExit = false;
9866 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
9867
9868 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9869 return rcStrict;
9870}
9871
9872
9873/** @} */
9874
9875
9876/**
9877 * Checks if any expensive dtrace probes are enabled and we should go to the
9878 * debug loop.
9879 *
9880 * @returns true if we should use debug loop, false if not.
9881 */
9882static bool hmR0VmxAnyExpensiveProbesEnabled(void)
9883{
9884 /* It's probably faster to OR the raw 32-bit counter variables together.
9885 Since the variables are in an array and the probes are next to one
9886 another (more or less), we have good locality. So, better read
9887 eight-nine cache lines ever time and only have one conditional, than
9888 128+ conditionals, right? */
9889 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
9890 | VBOXVMM_XCPT_DE_ENABLED_RAW()
9891 | VBOXVMM_XCPT_DB_ENABLED_RAW()
9892 | VBOXVMM_XCPT_BP_ENABLED_RAW()
9893 | VBOXVMM_XCPT_OF_ENABLED_RAW()
9894 | VBOXVMM_XCPT_BR_ENABLED_RAW()
9895 | VBOXVMM_XCPT_UD_ENABLED_RAW()
9896 | VBOXVMM_XCPT_NM_ENABLED_RAW()
9897 | VBOXVMM_XCPT_DF_ENABLED_RAW()
9898 | VBOXVMM_XCPT_TS_ENABLED_RAW()
9899 | VBOXVMM_XCPT_NP_ENABLED_RAW()
9900 | VBOXVMM_XCPT_SS_ENABLED_RAW()
9901 | VBOXVMM_XCPT_GP_ENABLED_RAW()
9902 | VBOXVMM_XCPT_PF_ENABLED_RAW()
9903 | VBOXVMM_XCPT_MF_ENABLED_RAW()
9904 | VBOXVMM_XCPT_AC_ENABLED_RAW()
9905 | VBOXVMM_XCPT_XF_ENABLED_RAW()
9906 | VBOXVMM_XCPT_VE_ENABLED_RAW()
9907 | VBOXVMM_XCPT_SX_ENABLED_RAW()
9908 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
9909 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
9910 ) != 0
9911 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
9912 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
9913 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
9914 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
9915 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
9916 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
9917 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
9918 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
9919 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
9920 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
9921 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
9922 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
9923 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
9924 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
9925 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
9926 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
9927 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
9928 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
9929 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
9930 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
9931 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
9932 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
9933 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
9934 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
9935 | VBOXVMM_INSTR_STR_ENABLED_RAW()
9936 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
9937 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
9938 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
9939 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
9940 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
9941 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
9942 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
9943 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
9944 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
9945 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
9946 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
9947 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
9948 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
9949 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
9950 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
9951 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
9952 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
9953 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
9954 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
9955 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
9956 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
9957 ) != 0
9958 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
9959 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
9960 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
9961 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
9962 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
9963 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
9964 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
9965 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
9966 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
9967 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
9968 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
9969 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
9970 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
9971 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
9972 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
9973 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
9974 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
9975 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
9976 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
9977 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
9978 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
9979 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
9980 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
9981 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
9982 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
9983 | VBOXVMM_EXIT_STR_ENABLED_RAW()
9984 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
9985 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
9986 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
9987 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
9988 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
9989 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
9990 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
9991 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
9992 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
9993 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
9994 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
9995 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
9996 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
9997 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
9998 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
9999 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10000 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10001 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10002 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10003 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10004 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10005 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10006 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10007 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10008 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10009 ) != 0;
10010}
10011
10012
10013/**
10014 * Runs the guest code using VT-x.
10015 *
10016 * @returns Strict VBox status code (i.e. informational status codes too).
10017 * @param pVCpu The cross context virtual CPU structure.
10018 */
10019VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
10020{
10021 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10022 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10023 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10024 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
10025
10026 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10027
10028 VBOXSTRICTRC rcStrict;
10029 if ( !pVCpu->hm.s.fUseDebugLoop
10030 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10031 && !DBGFIsStepping(pVCpu)
10032 && !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
10033 rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu);
10034 else
10035 rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu);
10036
10037 if (rcStrict == VERR_EM_INTERPRETER)
10038 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10039 else if (rcStrict == VINF_EM_RESET)
10040 rcStrict = VINF_EM_TRIPLE_FAULT;
10041
10042 int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
10043 if (RT_FAILURE(rc2))
10044 {
10045 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10046 rcStrict = rc2;
10047 }
10048 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10049 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10050 return rcStrict;
10051}
10052
10053
10054#ifndef HMVMX_USE_FUNCTION_TABLE
10055DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10056{
10057#ifdef DEBUG_ramshankar
10058#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
10059 do { \
10060 if (a_fSave != 0) \
10061 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
10062 VBOXSTRICTRC rcStrict = a_CallExpr; \
10063 if (a_fSave != 0) \
10064 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
10065 return rcStrict; \
10066 } while (0)
10067#else
10068# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
10069#endif
10070 switch (rcReason)
10071 {
10072 case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
10073 case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
10074 case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
10075 case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
10076 case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
10077 case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
10078 case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
10079 case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
10080 case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
10081 case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
10082 case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
10083 case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
10084 case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
10085 case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
10086 case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
10087 case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
10088 case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
10089 case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
10090 case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
10091 case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
10092 case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
10093 case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
10094 case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
10095 case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pVmxTransient));
10096 case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
10097 case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
10098 case VMX_EXIT_XDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10099 case VMX_EXIT_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10100 case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
10101 case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
10102 case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
10103 case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
10104 case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
10105 case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
10106
10107 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
10108 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
10109 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pVmxTransient);
10110 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pVmxTransient);
10111 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pVmxTransient);
10112 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pVmxTransient);
10113 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pVmxTransient);
10114 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
10115 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pVmxTransient);
10116
10117 case VMX_EXIT_VMCLEAR:
10118 case VMX_EXIT_VMLAUNCH:
10119 case VMX_EXIT_VMPTRLD:
10120 case VMX_EXIT_VMPTRST:
10121 case VMX_EXIT_VMREAD:
10122 case VMX_EXIT_VMRESUME:
10123 case VMX_EXIT_VMWRITE:
10124 case VMX_EXIT_VMXOFF:
10125 case VMX_EXIT_VMXON:
10126 case VMX_EXIT_INVEPT:
10127 case VMX_EXIT_INVVPID:
10128 case VMX_EXIT_VMFUNC:
10129 case VMX_EXIT_XSAVES:
10130 case VMX_EXIT_XRSTORS:
10131 return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
10132
10133 case VMX_EXIT_ENCLS:
10134 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10135 case VMX_EXIT_PML_FULL:
10136 default:
10137 return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
10138 }
10139#undef VMEXIT_CALL_RET
10140}
10141#endif /* !HMVMX_USE_FUNCTION_TABLE */
10142
10143
10144#ifdef VBOX_STRICT
10145/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10146# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10147 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10148
10149# define HMVMX_ASSERT_PREEMPT_CPUID() \
10150 do { \
10151 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10152 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10153 } while (0)
10154
10155# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10156 do { \
10157 AssertPtr((a_pVCpu)); \
10158 AssertPtr((a_pVmxTransient)); \
10159 Assert((a_pVmxTransient)->fVMEntryFailed == false); \
10160 Assert(ASMIntAreEnabled()); \
10161 HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10162 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10163 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
10164 HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10165 if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
10166 HMVMX_ASSERT_PREEMPT_CPUID(); \
10167 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10168 } while (0)
10169
10170# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10171 do { \
10172 Log4Func(("\n")); \
10173 } while (0)
10174#else
10175# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10176 do { \
10177 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10178 NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
10179 } while (0)
10180# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
10181#endif
10182
10183
10184/**
10185 * Advances the guest RIP by the specified number of bytes.
10186 *
10187 * @param pVCpu The cross context virtual CPU structure.
10188 * @param cbInstr Number of bytes to advance the RIP by.
10189 *
10190 * @remarks No-long-jump zone!!!
10191 */
10192DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
10193{
10194 /* Advance the RIP. */
10195 pVCpu->cpum.GstCtx.rip += cbInstr;
10196 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
10197
10198 /* Update interrupt inhibition. */
10199 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10200 && pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
10201 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10202}
10203
10204
10205/**
10206 * Advances the guest RIP after reading it from the VMCS.
10207 *
10208 * @returns VBox status code, no informational status codes.
10209 * @param pVCpu The cross context virtual CPU structure.
10210 * @param pVmxTransient Pointer to the VMX transient structure.
10211 *
10212 * @remarks No-long-jump zone!!!
10213 */
10214static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10215{
10216 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10217 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS);
10218 AssertRCReturn(rc, rc);
10219
10220 hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
10221
10222 /*
10223 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10224 * pending debug exception field as it takes care of priority of events.
10225 *
10226 * See Intel spec. 32.2.1 "Debug Exceptions".
10227 */
10228 if ( !pVCpu->hm.s.fSingleInstruction
10229 && pVCpu->cpum.GstCtx.eflags.Bits.u1TF)
10230 {
10231 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10232 AssertRCReturn(rc, rc);
10233 }
10234
10235 return VINF_SUCCESS;
10236}
10237
10238
10239/**
10240 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10241 * and update error record fields accordingly.
10242 *
10243 * @return VMX_IGS_* return codes.
10244 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10245 * wrong with the guest state.
10246 *
10247 * @param pVCpu The cross context virtual CPU structure.
10248 *
10249 * @remarks This function assumes our cache of the VMCS controls
10250 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10251 */
10252static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu)
10253{
10254#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10255#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10256 uError = (err); \
10257 break; \
10258 } else do { } while (0)
10259
10260 int rc;
10261 PVM pVM = pVCpu->CTX_SUFF(pVM);
10262 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10263 uint32_t uError = VMX_IGS_ERROR;
10264 uint32_t u32Val;
10265 bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10266
10267 do
10268 {
10269 /*
10270 * CR0.
10271 */
10272 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10273 uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10274 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10275 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10276 if (fUnrestrictedGuest)
10277 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
10278
10279 uint32_t u32GuestCr0;
10280 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10281 AssertRCBreak(rc);
10282 HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10283 HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10284 if ( !fUnrestrictedGuest
10285 && (u32GuestCr0 & X86_CR0_PG)
10286 && !(u32GuestCr0 & X86_CR0_PE))
10287 {
10288 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10289 }
10290
10291 /*
10292 * CR4.
10293 */
10294 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10295 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10296
10297 uint32_t u32GuestCr4;
10298 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10299 AssertRCBreak(rc);
10300 HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10301 HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10302
10303 /*
10304 * IA32_DEBUGCTL MSR.
10305 */
10306 uint64_t u64Val;
10307 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10308 AssertRCBreak(rc);
10309 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10310 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10311 {
10312 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10313 }
10314 uint64_t u64DebugCtlMsr = u64Val;
10315
10316#ifdef VBOX_STRICT
10317 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10318 AssertRCBreak(rc);
10319 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10320#endif
10321 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
10322
10323 /*
10324 * RIP and RFLAGS.
10325 */
10326 uint32_t u32Eflags;
10327#if HC_ARCH_BITS == 64
10328 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10329 AssertRCBreak(rc);
10330 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10331 if ( !fLongModeGuest
10332 || !pCtx->cs.Attr.n.u1Long)
10333 {
10334 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10335 }
10336 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10337 * must be identical if the "IA-32e mode guest" VM-entry
10338 * control is 1 and CS.L is 1. No check applies if the
10339 * CPU supports 64 linear-address bits. */
10340
10341 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10342 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10343 AssertRCBreak(rc);
10344 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10345 VMX_IGS_RFLAGS_RESERVED);
10346 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10347 u32Eflags = u64Val;
10348#else
10349 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10350 AssertRCBreak(rc);
10351 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10352 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10353#endif
10354
10355 if ( fLongModeGuest
10356 || ( fUnrestrictedGuest
10357 && !(u32GuestCr0 & X86_CR0_PE)))
10358 {
10359 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10360 }
10361
10362 uint32_t u32EntryInfo;
10363 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10364 AssertRCBreak(rc);
10365 if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
10366 && VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
10367 {
10368 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10369 }
10370
10371 /*
10372 * 64-bit checks.
10373 */
10374#if HC_ARCH_BITS == 64
10375 if (fLongModeGuest)
10376 {
10377 HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10378 HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10379 }
10380
10381 if ( !fLongModeGuest
10382 && (u32GuestCr4 & X86_CR4_PCIDE))
10383 {
10384 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10385 }
10386
10387 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10388 * 51:32 beyond the processor's physical-address width are 0. */
10389
10390 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10391 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10392 {
10393 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10394 }
10395
10396 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10397 AssertRCBreak(rc);
10398 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10399
10400 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10401 AssertRCBreak(rc);
10402 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10403#endif
10404
10405 /*
10406 * PERF_GLOBAL MSR.
10407 */
10408 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
10409 {
10410 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10411 AssertRCBreak(rc);
10412 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10413 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10414 }
10415
10416 /*
10417 * PAT MSR.
10418 */
10419 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
10420 {
10421 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10422 AssertRCBreak(rc);
10423 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10424 for (unsigned i = 0; i < 8; i++)
10425 {
10426 uint8_t u8Val = (u64Val & 0xff);
10427 if ( u8Val != 0 /* UC */
10428 && u8Val != 1 /* WC */
10429 && u8Val != 4 /* WT */
10430 && u8Val != 5 /* WP */
10431 && u8Val != 6 /* WB */
10432 && u8Val != 7 /* UC- */)
10433 {
10434 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10435 }
10436 u64Val >>= 8;
10437 }
10438 }
10439
10440 /*
10441 * EFER MSR.
10442 */
10443 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
10444 {
10445 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10446 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10447 AssertRCBreak(rc);
10448 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10449 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10450 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10451 & VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
10452 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10453 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10454 || !(u32GuestCr0 & X86_CR0_PG)
10455 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10456 VMX_IGS_EFER_LMA_LME_MISMATCH);
10457 }
10458
10459 /*
10460 * Segment registers.
10461 */
10462 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10463 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10464 if (!(u32Eflags & X86_EFL_VM))
10465 {
10466 /* CS */
10467 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10468 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10469 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10470 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10471 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10472 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10473 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10474 /* CS cannot be loaded with NULL in protected mode. */
10475 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10476 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10477 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10478 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10479 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10480 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10481 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10482 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10483 else
10484 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10485
10486 /* SS */
10487 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10488 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10489 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10490 if ( !(pCtx->cr0 & X86_CR0_PE)
10491 || pCtx->cs.Attr.n.u4Type == 3)
10492 {
10493 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10494 }
10495 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10496 {
10497 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10498 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10499 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10500 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10501 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10502 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10503 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10504 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10505 }
10506
10507 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmenReg(). */
10508 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10509 {
10510 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10511 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10512 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10513 || pCtx->ds.Attr.n.u4Type > 11
10514 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10515 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10516 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10517 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10518 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10519 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10520 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10521 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10522 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10523 }
10524 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10525 {
10526 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10527 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10528 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10529 || pCtx->es.Attr.n.u4Type > 11
10530 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10531 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10532 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10533 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10534 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10535 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10536 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10537 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10538 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10539 }
10540 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10541 {
10542 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10543 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10544 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10545 || pCtx->fs.Attr.n.u4Type > 11
10546 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10547 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10548 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10549 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10550 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10551 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10552 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10553 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10554 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10555 }
10556 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10557 {
10558 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10559 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10560 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10561 || pCtx->gs.Attr.n.u4Type > 11
10562 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10563 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10564 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10565 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10566 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10567 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10568 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10569 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10570 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10571 }
10572 /* 64-bit capable CPUs. */
10573#if HC_ARCH_BITS == 64
10574 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10575 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10576 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10577 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10578 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10579 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10580 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10581 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10582 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10583 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10584 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10585#endif
10586 }
10587 else
10588 {
10589 /* V86 mode checks. */
10590 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10591 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10592 {
10593 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10594 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10595 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10596 }
10597 else
10598 {
10599 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10600 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10601 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10602 }
10603
10604 /* CS */
10605 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10606 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10607 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10608 /* SS */
10609 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10610 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10611 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10612 /* DS */
10613 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10614 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10615 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10616 /* ES */
10617 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10618 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10619 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10620 /* FS */
10621 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10622 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10623 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10624 /* GS */
10625 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10626 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10627 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10628 /* 64-bit capable CPUs. */
10629#if HC_ARCH_BITS == 64
10630 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10631 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10632 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10633 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10634 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10635 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10636 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10637 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10638 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10639 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10640 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10641#endif
10642 }
10643
10644 /*
10645 * TR.
10646 */
10647 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10648 /* 64-bit capable CPUs. */
10649#if HC_ARCH_BITS == 64
10650 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10651#endif
10652 if (fLongModeGuest)
10653 {
10654 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10655 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10656 }
10657 else
10658 {
10659 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10660 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10661 VMX_IGS_TR_ATTR_TYPE_INVALID);
10662 }
10663 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10664 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10665 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10666 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10667 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10668 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10669 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10670 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10671
10672 /*
10673 * GDTR and IDTR.
10674 */
10675#if HC_ARCH_BITS == 64
10676 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10677 AssertRCBreak(rc);
10678 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10679
10680 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10681 AssertRCBreak(rc);
10682 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10683#endif
10684
10685 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10686 AssertRCBreak(rc);
10687 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10688
10689 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10690 AssertRCBreak(rc);
10691 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10692
10693 /*
10694 * Guest Non-Register State.
10695 */
10696 /* Activity State. */
10697 uint32_t u32ActivityState;
10698 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10699 AssertRCBreak(rc);
10700 HMVMX_CHECK_BREAK( !u32ActivityState
10701 || (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
10702 VMX_IGS_ACTIVITY_STATE_INVALID);
10703 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10704 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10705 uint32_t u32IntrState;
10706 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
10707 AssertRCBreak(rc);
10708 if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
10709 || u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10710 {
10711 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10712 }
10713
10714 /** @todo Activity state and injecting interrupts. Left as a todo since we
10715 * currently don't use activity states but ACTIVE. */
10716
10717 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
10718 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10719
10720 /* Guest interruptibility-state. */
10721 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10722 HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
10723 != (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10724 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10725 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10726 || !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
10727 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10728 if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
10729 {
10730 if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
10731 {
10732 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10733 && !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10734 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10735 }
10736 else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
10737 {
10738 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10739 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10740 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
10741 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10742 }
10743 }
10744 /** @todo Assumes the processor is not in SMM. */
10745 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
10746 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10747 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
10748 || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
10749 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10750 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
10751 && VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
10752 && VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
10753 {
10754 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
10755 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10756 }
10757
10758 /* Pending debug exceptions. */
10759#if HC_ARCH_BITS == 64
10760 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
10761 AssertRCBreak(rc);
10762 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10763 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10764 u32Val = u64Val; /* For pending debug exceptions checks below. */
10765#else
10766 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
10767 AssertRCBreak(rc);
10768 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10769 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10770#endif
10771
10772 if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10773 || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
10774 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10775 {
10776 if ( (u32Eflags & X86_EFL_TF)
10777 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10778 {
10779 /* Bit 14 is PendingDebug.BS. */
10780 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10781 }
10782 if ( !(u32Eflags & X86_EFL_TF)
10783 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10784 {
10785 /* Bit 14 is PendingDebug.BS. */
10786 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10787 }
10788 }
10789
10790 /* VMCS link pointer. */
10791 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10792 AssertRCBreak(rc);
10793 if (u64Val != UINT64_C(0xffffffffffffffff))
10794 {
10795 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10796 /** @todo Bits beyond the processor's physical-address width MBZ. */
10797 /** @todo 32-bit located in memory referenced by value of this field (as a
10798 * physical address) must contain the processor's VMCS revision ID. */
10799 /** @todo SMM checks. */
10800 }
10801
10802 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10803 * not using Nested Paging? */
10804 if ( pVM->hm.s.fNestedPaging
10805 && !fLongModeGuest
10806 && CPUMIsGuestInPAEModeEx(pCtx))
10807 {
10808 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10809 AssertRCBreak(rc);
10810 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10811
10812 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10813 AssertRCBreak(rc);
10814 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10815
10816 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10817 AssertRCBreak(rc);
10818 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10819
10820 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10821 AssertRCBreak(rc);
10822 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10823 }
10824
10825 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10826 if (uError == VMX_IGS_ERROR)
10827 uError = VMX_IGS_REASON_NOT_FOUND;
10828 } while (0);
10829
10830 pVCpu->hm.s.u32HMError = uError;
10831 return uError;
10832
10833#undef HMVMX_ERROR_BREAK
10834#undef HMVMX_CHECK_BREAK
10835}
10836
10837/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10838/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
10839/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10840
10841/** @name VM-exit handlers.
10842 * @{
10843 */
10844
10845/**
10846 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
10847 */
10848HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10849{
10850 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10851 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
10852 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
10853 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
10854 return VINF_SUCCESS;
10855 return VINF_EM_RAW_INTERRUPT;
10856}
10857
10858
10859/**
10860 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
10861 */
10862HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10863{
10864 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10865 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
10866
10867 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10868 AssertRCReturn(rc, rc);
10869
10870 uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
10871 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
10872 && uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
10873 Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
10874
10875 if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
10876 {
10877 /*
10878 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
10879 * injected it ourselves and anything we inject is not going to cause a VM-exit directly
10880 * for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
10881 *
10882 * [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
10883 * [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
10884 */
10885 VMXDispatchHostNmi();
10886 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
10887 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10888 return VINF_SUCCESS;
10889 }
10890
10891 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10892 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
10893 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
10894 { /* likely */ }
10895 else
10896 {
10897 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
10898 rcStrictRc1 = VINF_SUCCESS;
10899 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10900 return rcStrictRc1;
10901 }
10902
10903 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
10904 uint32_t uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
10905 switch (uIntType)
10906 {
10907 case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
10908 Assert(uVector == X86_XCPT_DB);
10909 RT_FALL_THRU();
10910 case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
10911 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
10912 RT_FALL_THRU();
10913 case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
10914 {
10915 /*
10916 * If there's any exception caused as a result of event injection, the resulting
10917 * secondary/final execption will be pending, we shall continue guest execution
10918 * after injecting the event. The page-fault case is complicated and we manually
10919 * handle any currently pending event in hmR0VmxExitXcptPF.
10920 */
10921 if (!pVCpu->hm.s.Event.fPending)
10922 { /* likely */ }
10923 else if (uVector != X86_XCPT_PF)
10924 {
10925 rc = VINF_SUCCESS;
10926 break;
10927 }
10928
10929 switch (uVector)
10930 {
10931 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
10932 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
10933 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
10934 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
10935 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
10936 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
10937
10938 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10939 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10940 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
10941 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10942 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
10943 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10944 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
10945 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10946 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
10947 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10948 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
10949 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10950 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
10951 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10952 default:
10953 {
10954 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
10955 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10956 {
10957 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
10958 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
10959 Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
10960
10961 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
10962 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10963 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10964 AssertRCReturn(rc, rc);
10965 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
10966 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
10967 0 /* GCPtrFaultAddress */);
10968 }
10969 else
10970 {
10971 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
10972 pVCpu->hm.s.u32HMError = uVector;
10973 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10974 }
10975 break;
10976 }
10977 }
10978 break;
10979 }
10980
10981 default:
10982 {
10983 pVCpu->hm.s.u32HMError = uExitIntInfo;
10984 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
10985 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
10986 break;
10987 }
10988 }
10989 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10990 return rc;
10991}
10992
10993
10994/**
10995 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
10996 */
10997HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10998{
10999 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11000
11001 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11002 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11003
11004 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11005 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11006 return VINF_SUCCESS;
11007}
11008
11009
11010/**
11011 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11012 */
11013HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11014{
11015 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11016 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)))
11017 {
11018 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11019 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11020 }
11021
11022 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11023
11024 /*
11025 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11026 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11027 */
11028 uint32_t fIntrState = 0;
11029 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
11030 AssertRCReturn(rc, rc);
11031
11032 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
11033 if ( fBlockSti
11034 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11035 {
11036 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11037 }
11038
11039 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11040 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11041
11042 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11043 return VINF_SUCCESS;
11044}
11045
11046
11047/**
11048 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11049 */
11050HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11051{
11052 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11053 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11054}
11055
11056
11057/**
11058 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11059 */
11060HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11061{
11062 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11063 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11064}
11065
11066
11067/**
11068 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11069 */
11070HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11071{
11072 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11073
11074 /*
11075 * Get the state we need and update the exit history entry.
11076 */
11077 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11078 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX);
11079 AssertRCReturn(rc, rc);
11080
11081 VBOXSTRICTRC rcStrict;
11082 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11083 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
11084 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11085 if (!pExitRec)
11086 {
11087 /*
11088 * Regular CPUID instruction execution.
11089 */
11090 rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
11091 if (rcStrict == VINF_SUCCESS)
11092 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_RAX
11093 | HM_CHANGED_GUEST_RCX | HM_CHANGED_GUEST_RDX | HM_CHANGED_GUEST_RBX);
11094 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11095 {
11096 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11097 rcStrict = VINF_SUCCESS;
11098 }
11099 }
11100 else
11101 {
11102 /*
11103 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
11104 */
11105 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11106 AssertRCReturn(rc2, rc2);
11107
11108 Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11109 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11110
11111 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11112 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
11113
11114 Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11115 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11116 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11117 }
11118 return rcStrict;
11119}
11120
11121
11122/**
11123 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11124 */
11125HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11126{
11127 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11128 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4);
11129 AssertRCReturn(rc, rc);
11130
11131 if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
11132 return VINF_EM_RAW_EMULATE_INSTR;
11133
11134 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11135 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11136}
11137
11138
11139/**
11140 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11141 */
11142HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11143{
11144 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11145 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11146 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11147 AssertRCReturn(rc, rc);
11148
11149 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11150 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11151 {
11152 /* If we get a spurious VM-exit when offsetting is enabled,
11153 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11154 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
11155 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11156 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11157 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX);
11158 }
11159 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11160 {
11161 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11162 rcStrict = VINF_SUCCESS;
11163 }
11164 return rcStrict;
11165}
11166
11167
11168/**
11169 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11170 */
11171HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11172{
11173 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11174 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_TSC_AUX);
11175 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11176 AssertRCReturn(rc, rc);
11177
11178 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11179 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11180 {
11181 /* If we get a spurious VM-exit when offsetting is enabled,
11182 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11183 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
11184 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11185 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11186 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX | HM_CHANGED_GUEST_RCX);
11187 }
11188 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11189 {
11190 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11191 rcStrict = VINF_SUCCESS;
11192 }
11193 return rcStrict;
11194}
11195
11196
11197/**
11198 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11199 */
11200HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11201{
11202 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11203 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11204 AssertRCReturn(rc, rc);
11205
11206 PVM pVM = pVCpu->CTX_SUFF(pVM);
11207 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11208 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11209 if (RT_LIKELY(rc == VINF_SUCCESS))
11210 {
11211 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11212 Assert(pVmxTransient->cbInstr == 2);
11213 }
11214 else
11215 {
11216 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11217 rc = VERR_EM_INTERPRETER;
11218 }
11219 return rc;
11220}
11221
11222
11223/**
11224 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11225 */
11226HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11227{
11228 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11229
11230 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11231 if (EMAreHypercallInstructionsEnabled(pVCpu))
11232 {
11233 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_SS
11234 | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_EFER);
11235 AssertRCReturn(rc, rc);
11236
11237 /* Perform the hypercall. */
11238 rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
11239 if (rcStrict == VINF_SUCCESS)
11240 {
11241 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11242 AssertRCReturn(rc, rc);
11243 }
11244 else
11245 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11246 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11247 || RT_FAILURE(rcStrict));
11248
11249 /* If the hypercall changes anything other than guest's general-purpose registers,
11250 we would need to reload the guest changed bits here before VM-entry. */
11251 }
11252 else
11253 Log4Func(("Hypercalls not enabled\n"));
11254
11255 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11256 if (RT_FAILURE(rcStrict))
11257 {
11258 hmR0VmxSetPendingXcptUD(pVCpu);
11259 rcStrict = VINF_SUCCESS;
11260 }
11261
11262 return rcStrict;
11263}
11264
11265
11266/**
11267 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11268 */
11269HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11270{
11271 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11272 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11273
11274 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11275 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11276 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_DS);
11277 AssertRCReturn(rc, rc);
11278
11279 VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQualification);
11280
11281 if (rcStrict == VINF_SUCCESS || rcStrict == VINF_PGM_SYNC_CR3)
11282 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11283 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11284 {
11285 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11286 rcStrict = VINF_SUCCESS;
11287 }
11288 else
11289 AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) sttus: %Rrc\n",
11290 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11291 return rcStrict;
11292}
11293
11294
11295/**
11296 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11297 */
11298HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11299{
11300 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11301 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11302 AssertRCReturn(rc, rc);
11303
11304 PVM pVM = pVCpu->CTX_SUFF(pVM);
11305 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11306 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11307 if (RT_LIKELY(rc == VINF_SUCCESS))
11308 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11309 else
11310 {
11311 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11312 rc = VERR_EM_INTERPRETER;
11313 }
11314 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11315 return rc;
11316}
11317
11318
11319/**
11320 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11321 */
11322HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11323{
11324 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11325 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11326 AssertRCReturn(rc, rc);
11327
11328 PVM pVM = pVCpu->CTX_SUFF(pVM);
11329 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11330 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11331 rc = VBOXSTRICTRC_VAL(rc2);
11332 if (RT_LIKELY( rc == VINF_SUCCESS
11333 || rc == VINF_EM_HALT))
11334 {
11335 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11336 AssertRCReturn(rc3, rc3);
11337
11338 if ( rc == VINF_EM_HALT
11339 && EMMonitorWaitShouldContinue(pVCpu, pCtx))
11340 rc = VINF_SUCCESS;
11341 }
11342 else
11343 {
11344 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11345 rc = VERR_EM_INTERPRETER;
11346 }
11347 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11348 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11349 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11350 return rc;
11351}
11352
11353
11354/**
11355 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11356 */
11357HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11358{
11359 /*
11360 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
11361 * mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
11362 * treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
11363 * VMX root operation. If we get here, something funny is going on.
11364 *
11365 * See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
11366 */
11367 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11368 AssertMsgFailed(("Unexpected RSM VM-exit\n"));
11369 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11370}
11371
11372
11373/**
11374 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11375 */
11376HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11377{
11378 /*
11379 * This can only happen if we support dual-monitor treatment of SMI, which can be activated
11380 * by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
11381 * this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
11382 * an SMI. If we get here, something funny is going on.
11383 *
11384 * See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11385 * See Intel spec. 25.3 "Other Causes of VM-Exits"
11386 */
11387 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11388 AssertMsgFailed(("Unexpected SMI VM-exit\n"));
11389 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11390}
11391
11392
11393/**
11394 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11395 */
11396HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11397{
11398 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11399 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11400 AssertMsgFailed(("Unexpected IO SMI VM-exit\n"));
11401 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11402}
11403
11404
11405/**
11406 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11407 */
11408HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11409{
11410 /*
11411 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
11412 * We don't make use of it as our guests don't have direct access to the host LAPIC.
11413 * See Intel spec. 25.3 "Other Causes of VM-exits".
11414 */
11415 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11416 AssertMsgFailed(("Unexpected SIPI VM-exit\n"));
11417 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11418}
11419
11420
11421/**
11422 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11423 * VM-exit.
11424 */
11425HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11426{
11427 /*
11428 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11429 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11430 *
11431 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11432 * See Intel spec. "23.8 Restrictions on VMX operation".
11433 */
11434 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11435 return VINF_SUCCESS;
11436}
11437
11438
11439/**
11440 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11441 * VM-exit.
11442 */
11443HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11444{
11445 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11446 return VINF_EM_RESET;
11447}
11448
11449
11450/**
11451 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11452 */
11453HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11454{
11455 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11456 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_HLT_EXIT);
11457
11458 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11459 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
11460 AssertRCReturn(rc, rc);
11461
11462 if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
11463 rc = VINF_SUCCESS;
11464 else
11465 rc = VINF_EM_HALT;
11466
11467 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11468 if (rc != VINF_SUCCESS)
11469 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11470 return rc;
11471}
11472
11473
11474/**
11475 * VM-exit handler for instructions that result in a \#UD exception delivered to
11476 * the guest.
11477 */
11478HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11479{
11480 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11481 hmR0VmxSetPendingXcptUD(pVCpu);
11482 return VINF_SUCCESS;
11483}
11484
11485
11486/**
11487 * VM-exit handler for expiry of the VMX preemption timer.
11488 */
11489HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11490{
11491 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11492
11493 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11494 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11495
11496 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11497 PVM pVM = pVCpu->CTX_SUFF(pVM);
11498 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11499 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11500 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11501}
11502
11503
11504/**
11505 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11506 */
11507HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11508{
11509 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11510
11511 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11512 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_CR4);
11513 AssertRCReturn(rc, rc);
11514
11515 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11516 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11517 : HM_CHANGED_RAISED_XCPT_MASK);
11518
11519 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11520 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
11521
11522 return rcStrict;
11523}
11524
11525
11526/**
11527 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11528 */
11529HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11530{
11531 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11532 /** @todo Use VM-exit instruction information. */
11533 return VERR_EM_INTERPRETER;
11534}
11535
11536
11537/**
11538 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11539 * Error VM-exit.
11540 */
11541HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11542{
11543 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11544 AssertRCReturn(rc, rc);
11545 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11546 if (RT_FAILURE(rc))
11547 return rc;
11548
11549 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
11550 NOREF(uInvalidReason);
11551
11552#ifdef VBOX_STRICT
11553 uint32_t fIntrState;
11554 RTHCUINTREG uHCReg;
11555 uint64_t u64Val;
11556 uint32_t u32Val;
11557
11558 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11559 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11560 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11561 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
11562 AssertRCReturn(rc, rc);
11563
11564 Log4(("uInvalidReason %u\n", uInvalidReason));
11565 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11566 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11567 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11568 Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
11569
11570 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11571 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11572 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11573 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11574 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11575 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11576 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11577 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11578 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11579 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11580 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11581 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11582
11583 hmR0DumpRegs(pVCpu);
11584#else
11585 NOREF(pVmxTransient);
11586#endif
11587
11588 return VERR_VMX_INVALID_GUEST_STATE;
11589}
11590
11591
11592/**
11593 * VM-exit handler for VM-entry failure due to an MSR-load
11594 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11595 */
11596HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11597{
11598 AssertMsgFailed(("Unexpected MSR-load exit\n"));
11599 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11600}
11601
11602
11603/**
11604 * VM-exit handler for VM-entry failure due to a machine-check event
11605 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11606 */
11607HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11608{
11609 AssertMsgFailed(("Unexpected machine-check event exit\n"));
11610 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11611}
11612
11613
11614/**
11615 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11616 * theory.
11617 */
11618HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11619{
11620 RT_NOREF2(pVCpu, pVmxTransient);
11621 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d\n", pVmxTransient->uExitReason));
11622 return VERR_VMX_UNDEFINED_EXIT_CODE;
11623}
11624
11625
11626/**
11627 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11628 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11629 * Conditional VM-exit.
11630 */
11631HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11632{
11633 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11634
11635 /* By default, we don't enable VMX_PROC_CTLS2_DESCRIPTOR_TABLE_EXIT. */
11636 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11637 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_DESC_TABLE_EXIT)
11638 return VERR_EM_INTERPRETER;
11639 AssertMsgFailed(("Unexpected XDTR access\n"));
11640 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11641}
11642
11643
11644/**
11645 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11646 */
11647HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11648{
11649 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11650
11651 /* By default, we don't enable VMX_PROC_CTLS2_RDRAND_EXIT. */
11652 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_RDRAND_EXIT)
11653 return VERR_EM_INTERPRETER;
11654 AssertMsgFailed(("Unexpected RDRAND exit\n"));
11655 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11656}
11657
11658
11659/**
11660 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11661 */
11662HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11663{
11664 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11665
11666 /** @todo Optimize this: We currently drag in in the whole MSR state
11667 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11668 * MSRs required. That would require changes to IEM and possibly CPUM too.
11669 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11670 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; NOREF(idMsr); /* Save it. */
11671 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11672 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
11673 AssertRCReturn(rc, rc);
11674
11675 Log4Func(("ecx=%#RX32\n", idMsr));
11676
11677#ifdef VBOX_STRICT
11678 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
11679 {
11680 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr)
11681 && idMsr != MSR_K6_EFER)
11682 {
11683 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
11684 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11685 }
11686 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11687 {
11688 VMXMSREXITREAD enmRead;
11689 VMXMSREXITWRITE enmWrite;
11690 int rc2 = hmR0VmxGetMsrPermission(pVCpu, idMsr, &enmRead, &enmWrite);
11691 AssertRCReturn(rc2, rc2);
11692 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11693 {
11694 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
11695 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11696 }
11697 }
11698 }
11699#endif
11700
11701 VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
11702 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11703 if (rcStrict == VINF_SUCCESS)
11704 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11705 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX);
11706 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11707 {
11708 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11709 rcStrict = VINF_SUCCESS;
11710 }
11711 else
11712 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11713
11714 return rcStrict;
11715}
11716
11717
11718/**
11719 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11720 */
11721HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11722{
11723 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11724
11725 /** @todo Optimize this: We currently drag in in the whole MSR state
11726 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11727 * MSRs required. That would require changes to IEM and possibly CPUM too.
11728 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11729 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; /* Save it. */
11730 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11731 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
11732 AssertRCReturn(rc, rc);
11733
11734 Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
11735
11736 VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
11737 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
11738
11739 if (rcStrict == VINF_SUCCESS)
11740 {
11741 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11742
11743 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
11744 if ( idMsr == MSR_IA32_APICBASE
11745 || ( idMsr >= MSR_IA32_X2APIC_START
11746 && idMsr <= MSR_IA32_X2APIC_END))
11747 {
11748 /*
11749 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
11750 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before IEM changes it.
11751 */
11752 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11753 }
11754 else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
11755 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11756 else if (idMsr == MSR_K6_EFER)
11757 {
11758 /*
11759 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
11760 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
11761 * the other bits as well, SCE and NXE. See @bugref{7368}.
11762 */
11763 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS
11764 | HM_CHANGED_VMX_EXIT_CTLS);
11765 }
11766
11767 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
11768 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
11769 {
11770 switch (idMsr)
11771 {
11772 case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
11773 case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
11774 case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
11775 case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
11776 case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
11777 case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
11778 default:
11779 {
11780 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
11781 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
11782 else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11783 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
11784 break;
11785 }
11786 }
11787 }
11788#ifdef VBOX_STRICT
11789 else
11790 {
11791 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
11792 switch (idMsr)
11793 {
11794 case MSR_IA32_SYSENTER_CS:
11795 case MSR_IA32_SYSENTER_EIP:
11796 case MSR_IA32_SYSENTER_ESP:
11797 case MSR_K8_FS_BASE:
11798 case MSR_K8_GS_BASE:
11799 {
11800 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
11801 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11802 }
11803
11804 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
11805 default:
11806 {
11807 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
11808 {
11809 /* EFER writes are always intercepted, see hmR0VmxExportGuestMsrs(). */
11810 if (idMsr != MSR_K6_EFER)
11811 {
11812 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11813 idMsr));
11814 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11815 }
11816 }
11817
11818 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11819 {
11820 VMXMSREXITREAD enmRead;
11821 VMXMSREXITWRITE enmWrite;
11822 int rc2 = hmR0VmxGetMsrPermission(pVCpu, idMsr, &enmRead, &enmWrite);
11823 AssertRCReturn(rc2, rc2);
11824 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
11825 {
11826 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
11827 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11828 }
11829 }
11830 break;
11831 }
11832 }
11833 }
11834#endif /* VBOX_STRICT */
11835 }
11836 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11837 {
11838 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11839 rcStrict = VINF_SUCCESS;
11840 }
11841 else
11842 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11843
11844 return rcStrict;
11845}
11846
11847
11848/**
11849 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
11850 */
11851HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11852{
11853 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11854 /** @todo The guest has likely hit a contended spinlock. We might want to
11855 * poke a schedule different guest VCPU. */
11856 return VINF_EM_RAW_INTERRUPT;
11857}
11858
11859
11860/**
11861 * VM-exit handler for when the TPR value is lowered below the specified
11862 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
11863 */
11864HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11865{
11866 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11867 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
11868
11869 /*
11870 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
11871 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
11872 */
11873 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
11874 return VINF_SUCCESS;
11875}
11876
11877
11878/**
11879 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
11880 * VM-exit.
11881 *
11882 * @retval VINF_SUCCESS when guest execution can continue.
11883 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
11884 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
11885 * interpreter.
11886 */
11887HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11888{
11889 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11890 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
11891
11892 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11893 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11894 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11895 AssertRCReturn(rc, rc);
11896
11897 VBOXSTRICTRC rcStrict;
11898 PVM pVM = pVCpu->CTX_SUFF(pVM);
11899 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
11900 uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQualification);
11901 switch (uAccessType)
11902 {
11903 case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
11904 {
11905 uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
11906 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
11907 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
11908 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification));
11909 AssertMsg( rcStrict == VINF_SUCCESS
11910 || rcStrict == VINF_IEM_RAISED_XCPT
11911 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11912
11913 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
11914 {
11915 case 0:
11916 {
11917 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
11918 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
11919 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
11920 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
11921
11922 /*
11923 * This is a kludge for handling switches back to real mode when we try to use
11924 * V86 mode to run real mode code directly. Problem is that V86 mode cannot
11925 * deal with special selector values, so we have to return to ring-3 and run
11926 * there till the selector values are V86 mode compatible.
11927 *
11928 * Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
11929 * latter is an alias for VINF_IEM_RAISED_XCPT which is converted to VINF_SUCCESs
11930 * at the end of this function.
11931 */
11932 if ( rc == VINF_SUCCESS
11933 && !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
11934 && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
11935 && (uOldCr0 & X86_CR0_PE)
11936 && !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
11937 {
11938 /** @todo check selectors rather than returning all the time. */
11939 Log4(("CRx CR0 write: back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
11940 rcStrict = VINF_EM_RESCHEDULE_REM;
11941 }
11942 break;
11943 }
11944
11945 case 2:
11946 {
11947 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
11948 /* Nothing to do here, CR2 it's not part of the VMCS. */
11949 break;
11950 }
11951
11952 case 3:
11953 {
11954 Assert( !pVM->hm.s.fNestedPaging
11955 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
11956 || pVCpu->hm.s.fUsingDebugLoop);
11957 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
11958 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
11959 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR3);
11960 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
11961 break;
11962 }
11963
11964 case 4:
11965 {
11966 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
11967 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
11968 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR4);
11969 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
11970 pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
11971 break;
11972 }
11973
11974 case 8:
11975 {
11976 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
11977 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
11978 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
11979 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_APIC_TPR);
11980 break;
11981 }
11982 default:
11983 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification)));
11984 break;
11985 }
11986 break;
11987 }
11988
11989 case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
11990 {
11991 Assert( !pVM->hm.s.fNestedPaging
11992 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
11993 || pVCpu->hm.s.fUsingDebugLoop
11994 || VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 3);
11995 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
11996 Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 8
11997 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
11998
11999 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12000 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification),
12001 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification));
12002 AssertMsg( rcStrict == VINF_SUCCESS
12003 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12004#ifdef VBOX_WITH_STATISTICS
12005 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
12006 {
12007 case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
12008 case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
12009 case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
12010 case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
12011 case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
12012 }
12013#endif
12014 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
12015 VBOXSTRICTRC_VAL(rcStrict)));
12016 if (VMX_EXIT_QUAL_CRX_GENREG(uExitQualification) == X86_GREG_xSP)
12017 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_RSP);
12018 else
12019 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12020 break;
12021 }
12022
12023 case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12024 {
12025 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12026 AssertMsg( rcStrict == VINF_SUCCESS
12027 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12028
12029 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12030 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12031 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12032 break;
12033 }
12034
12035 case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12036 {
12037 /* Note! LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here. */
12038 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12039 VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQualification));
12040 AssertMsg( rcStrict == VINF_SUCCESS
12041 || rcStrict == VINF_IEM_RAISED_XCPT
12042 , ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12043
12044 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12045 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12046 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12047 break;
12048 }
12049
12050 default:
12051 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12052 VERR_VMX_UNEXPECTED_EXCEPTION);
12053 }
12054
12055 Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS))
12056 == (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS));
12057 if (rcStrict == VINF_IEM_RAISED_XCPT)
12058 {
12059 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
12060 rcStrict = VINF_SUCCESS;
12061 }
12062
12063 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12064 NOREF(pVM);
12065 return rcStrict;
12066}
12067
12068
12069/**
12070 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12071 * VM-exit.
12072 */
12073HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12074{
12075 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12076 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12077
12078 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12079 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12080 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12081 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_EFER);
12082 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12083 AssertRCReturn(rc, rc);
12084
12085 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12086 uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQualification);
12087 uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQualification);
12088 bool fIOWrite = ( VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQualification)
12089 == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12090 bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQualification);
12091 bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
12092 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12093 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12094
12095 /*
12096 * Update exit history to see if this exit can be optimized.
12097 */
12098 VBOXSTRICTRC rcStrict;
12099 PCEMEXITREC pExitRec = NULL;
12100 if ( !fGstStepping
12101 && !fDbgStepping)
12102 pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12103 !fIOString
12104 ? !fIOWrite
12105 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12106 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12107 : !fIOWrite
12108 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12109 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12110 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12111 if (!pExitRec)
12112 {
12113 /* I/O operation lookup arrays. */
12114 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12115 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12116 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12117 uint32_t const cbInstr = pVmxTransient->cbInstr;
12118 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12119 PVM pVM = pVCpu->CTX_SUFF(pVM);
12120 if (fIOString)
12121 {
12122 /*
12123 * INS/OUTS - I/O String instruction.
12124 *
12125 * Use instruction-information if available, otherwise fall back on
12126 * interpreting the instruction.
12127 */
12128 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12129 AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
12130 bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
12131 if (fInsOutsInfo)
12132 {
12133 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12134 AssertRCReturn(rc2, rc2);
12135 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12136 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12137 IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12138 bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification);
12139 if (fIOWrite)
12140 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12141 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12142 else
12143 {
12144 /*
12145 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12146 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12147 * See Intel Instruction spec. for "INS".
12148 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12149 */
12150 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12151 }
12152 }
12153 else
12154 rcStrict = IEMExecOne(pVCpu);
12155
12156 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12157 fUpdateRipAlready = true;
12158 }
12159 else
12160 {
12161 /*
12162 * IN/OUT - I/O instruction.
12163 */
12164 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12165 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12166 Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification));
12167 if (fIOWrite)
12168 {
12169 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
12170 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12171 if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
12172 && !pCtx->eflags.Bits.u1TF)
12173 rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
12174 }
12175 else
12176 {
12177 uint32_t u32Result = 0;
12178 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12179 if (IOM_SUCCESS(rcStrict))
12180 {
12181 /* Save result of I/O IN instr. in AL/AX/EAX. */
12182 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12183 }
12184 if ( rcStrict == VINF_IOM_R3_IOPORT_READ
12185 && !pCtx->eflags.Bits.u1TF)
12186 rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
12187 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12188 }
12189 }
12190
12191 if (IOM_SUCCESS(rcStrict))
12192 {
12193 if (!fUpdateRipAlready)
12194 {
12195 hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
12196 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12197 }
12198
12199 /*
12200 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12201 * while booting Fedora 17 64-bit guest.
12202 *
12203 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12204 */
12205 if (fIOString)
12206 {
12207 /** @todo Single-step for INS/OUTS with REP prefix? */
12208 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12209 }
12210 else if ( !fDbgStepping
12211 && fGstStepping)
12212 {
12213 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12214 AssertRCReturn(rc, rc);
12215 }
12216
12217 /*
12218 * If any I/O breakpoints are armed, we need to check if one triggered
12219 * and take appropriate action.
12220 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12221 */
12222 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12223 AssertRCReturn(rc, rc);
12224
12225 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12226 * execution engines about whether hyper BPs and such are pending. */
12227 uint32_t const uDr7 = pCtx->dr[7];
12228 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12229 && X86_DR7_ANY_RW_IO(uDr7)
12230 && (pCtx->cr4 & X86_CR4_DE))
12231 || DBGFBpIsHwIoArmed(pVM)))
12232 {
12233 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12234
12235 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12236 VMMRZCallRing3Disable(pVCpu);
12237 HM_DISABLE_PREEMPT(pVCpu);
12238
12239 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12240
12241 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
12242 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12243 {
12244 /* Raise #DB. */
12245 if (fIsGuestDbgActive)
12246 ASMSetDR6(pCtx->dr[6]);
12247 if (pCtx->dr[7] != uDr7)
12248 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_DR7;
12249
12250 hmR0VmxSetPendingXcptDB(pVCpu);
12251 }
12252 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12253 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12254 else if ( rcStrict2 != VINF_SUCCESS
12255 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12256 rcStrict = rcStrict2;
12257 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12258
12259 HM_RESTORE_PREEMPT();
12260 VMMRZCallRing3Enable(pVCpu);
12261 }
12262 }
12263
12264#ifdef VBOX_STRICT
12265 if ( rcStrict == VINF_IOM_R3_IOPORT_READ
12266 || rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
12267 Assert(!fIOWrite);
12268 else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
12269 || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
12270 || rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
12271 Assert(fIOWrite);
12272 else
12273 {
12274# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12275 * statuses, that the VMM device and some others may return. See
12276 * IOM_SUCCESS() for guidance. */
12277 AssertMsg( RT_FAILURE(rcStrict)
12278 || rcStrict == VINF_SUCCESS
12279 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12280 || rcStrict == VINF_EM_DBG_BREAKPOINT
12281 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12282 || rcStrict == VINF_EM_RAW_TO_R3
12283 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12284# endif
12285 }
12286#endif
12287 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12288 }
12289 else
12290 {
12291 /*
12292 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12293 */
12294 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12295 AssertRCReturn(rc2, rc2);
12296 STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
12297 : fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
12298 Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12299 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12300 VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification) ? "REP " : "",
12301 fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
12302
12303 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12304 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12305
12306 Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12307 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12308 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12309 }
12310 return rcStrict;
12311}
12312
12313
12314/**
12315 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12316 * VM-exit.
12317 */
12318HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12319{
12320 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12321
12322 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12323 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12324 AssertRCReturn(rc, rc);
12325 if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12326 {
12327 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12328 AssertRCReturn(rc, rc);
12329 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12330 {
12331 uint32_t uErrCode;
12332 RTGCUINTPTR GCPtrFaultAddress;
12333 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12334 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12335 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
12336 if (fErrorCodeValid)
12337 {
12338 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12339 AssertRCReturn(rc, rc);
12340 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12341 }
12342 else
12343 uErrCode = 0;
12344
12345 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12346 && uVector == X86_XCPT_PF)
12347 GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
12348 else
12349 GCPtrFaultAddress = 0;
12350
12351 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12352 0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
12353
12354 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12355 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12356 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12357 }
12358 }
12359
12360 /* Fall back to the interpreter to emulate the task-switch. */
12361 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12362 return VERR_EM_INTERPRETER;
12363}
12364
12365
12366/**
12367 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12368 */
12369HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12370{
12371 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12372 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_MONITOR_TRAP_FLAG);
12373 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
12374 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12375 AssertRCReturn(rc, rc);
12376 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12377 return VINF_EM_DBG_STEPPED;
12378}
12379
12380
12381/**
12382 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12383 */
12384HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12385{
12386 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12387
12388 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12389
12390 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12391 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12392 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12393 {
12394 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12395 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12396 {
12397 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12398 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12399 }
12400 }
12401 else
12402 {
12403 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12404 rcStrict1 = VINF_SUCCESS;
12405 return rcStrict1;
12406 }
12407
12408 /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12409 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12410 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12411 AssertRCReturn(rc, rc);
12412
12413 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12414 uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12415 VBOXSTRICTRC rcStrict2;
12416 switch (uAccessType)
12417 {
12418 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12419 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12420 {
12421 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
12422 || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
12423 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12424
12425 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12426 GCPhys &= PAGE_BASE_GC_MASK;
12427 GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12428 PVM pVM = pVCpu->CTX_SUFF(pVM);
12429 Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12430 VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12431
12432 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12433 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12434 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12435 CPUMCTX2CORE(pCtx), GCPhys);
12436 Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12437 if ( rcStrict2 == VINF_SUCCESS
12438 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12439 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12440 {
12441 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
12442 | HM_CHANGED_GUEST_APIC_TPR);
12443 rcStrict2 = VINF_SUCCESS;
12444 }
12445 break;
12446 }
12447
12448 default:
12449 Log4Func(("uAccessType=%#x\n", uAccessType));
12450 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12451 break;
12452 }
12453
12454 if (rcStrict2 != VINF_SUCCESS)
12455 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12456 return rcStrict2;
12457}
12458
12459
12460/**
12461 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12462 * VM-exit.
12463 */
12464HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12465{
12466 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12467
12468 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12469 if (pVmxTransient->fWasGuestDebugStateActive)
12470 {
12471 AssertMsgFailed(("Unexpected MOV DRx exit\n"));
12472 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12473 }
12474
12475 if ( !pVCpu->hm.s.fSingleInstruction
12476 && !pVmxTransient->fWasHyperDebugStateActive)
12477 {
12478 Assert(!DBGFIsStepping(pVCpu));
12479 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12480
12481 /* Don't intercept MOV DRx any more. */
12482 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
12483 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12484 AssertRCReturn(rc, rc);
12485
12486 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12487 VMMRZCallRing3Disable(pVCpu);
12488 HM_DISABLE_PREEMPT(pVCpu);
12489
12490 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12491 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12492 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12493
12494 HM_RESTORE_PREEMPT();
12495 VMMRZCallRing3Enable(pVCpu);
12496
12497#ifdef VBOX_WITH_STATISTICS
12498 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12499 AssertRCReturn(rc, rc);
12500 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12501 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12502 else
12503 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12504#endif
12505 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12506 return VINF_SUCCESS;
12507 }
12508
12509 /*
12510 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12511 * Update the segment registers and DR7 from the CPU.
12512 */
12513 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12514 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12515 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_DR7);
12516 AssertRCReturn(rc, rc);
12517 Log4Func(("CS:RIP=%04x:%08RX64\n", pCtx->cs.Sel, pCtx->rip));
12518
12519 PVM pVM = pVCpu->CTX_SUFF(pVM);
12520 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12521 {
12522 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12523 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification),
12524 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification));
12525 if (RT_SUCCESS(rc))
12526 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12527 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12528 }
12529 else
12530 {
12531 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12532 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification),
12533 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification));
12534 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12535 }
12536
12537 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12538 if (RT_SUCCESS(rc))
12539 {
12540 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12541 AssertRCReturn(rc2, rc2);
12542 return VINF_SUCCESS;
12543 }
12544 return rc;
12545}
12546
12547
12548/**
12549 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12550 * Conditional VM-exit.
12551 */
12552HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12553{
12554 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12555 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12556
12557 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12558 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12559 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12560 {
12561 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12562 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12563 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12564 {
12565 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12566 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12567 }
12568 }
12569 else
12570 {
12571 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12572 rcStrict1 = VINF_SUCCESS;
12573 return rcStrict1;
12574 }
12575
12576 /*
12577 * Get sufficent state and update the exit history entry.
12578 */
12579 RTGCPHYS GCPhys;
12580 int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
12581 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12582 AssertRCReturn(rc, rc);
12583
12584 VBOXSTRICTRC rcStrict;
12585 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12586 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
12587 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12588 if (!pExitRec)
12589 {
12590 /*
12591 * If we succeed, resume guest execution.
12592 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12593 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12594 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12595 * weird case. See @bugref{6043}.
12596 */
12597 PVM pVM = pVCpu->CTX_SUFF(pVM);
12598 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12599 rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
12600 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12601 if ( rcStrict == VINF_SUCCESS
12602 || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12603 || rcStrict == VERR_PAGE_NOT_PRESENT)
12604 {
12605 /* Successfully handled MMIO operation. */
12606 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
12607 | HM_CHANGED_GUEST_APIC_TPR);
12608 rcStrict = VINF_SUCCESS;
12609 }
12610 }
12611 else
12612 {
12613 /*
12614 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12615 */
12616 int rc2 = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12617 AssertRCReturn(rc2, rc2);
12618
12619 Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12620 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12621
12622 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12623 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12624
12625 Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12626 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12627 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12628 }
12629 return VBOXSTRICTRC_TODO(rcStrict);
12630}
12631
12632
12633/**
12634 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12635 * VM-exit.
12636 */
12637HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12638{
12639 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12640 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12641
12642 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12643 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12644 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12645 {
12646 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12647 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12648 Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12649 }
12650 else
12651 {
12652 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12653 rcStrict1 = VINF_SUCCESS;
12654 return rcStrict1;
12655 }
12656
12657 RTGCPHYS GCPhys;
12658 int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
12659 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12660 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12661 AssertRCReturn(rc, rc);
12662
12663 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12664 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12665
12666 RTGCUINT uErrorCode = 0;
12667 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
12668 uErrorCode |= X86_TRAP_PF_ID;
12669 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_DATA_WRITE)
12670 uErrorCode |= X86_TRAP_PF_RW;
12671 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
12672 uErrorCode |= X86_TRAP_PF_P;
12673
12674 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12675
12676
12677 /* Handle the pagefault trap for the nested shadow table. */
12678 PVM pVM = pVCpu->CTX_SUFF(pVM);
12679 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12680
12681 Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12682 uErrorCode, pCtx->cs.Sel, pCtx->rip));
12683
12684 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
12685 TRPMResetTrap(pVCpu);
12686
12687 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12688 if ( rcStrict2 == VINF_SUCCESS
12689 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12690 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12691 {
12692 /* Successfully synced our nested page tables. */
12693 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12694 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
12695 return VINF_SUCCESS;
12696 }
12697
12698 Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12699 return rcStrict2;
12700}
12701
12702/** @} */
12703
12704/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12705/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
12706/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12707
12708/** @name VM-exit exception handlers.
12709 * @{
12710 */
12711
12712/**
12713 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12714 */
12715static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12716{
12717 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12718 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12719
12720 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
12721 AssertRCReturn(rc, rc);
12722
12723 if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
12724 {
12725 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12726 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12727
12728 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12729 * provides VM-exit instruction length. If this causes problem later,
12730 * disassemble the instruction like it's done on AMD-V. */
12731 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12732 AssertRCReturn(rc2, rc2);
12733 return rc;
12734 }
12735
12736 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12737 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12738 return rc;
12739}
12740
12741
12742/**
12743 * VM-exit exception handler for \#BP (Breakpoint exception).
12744 */
12745static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12746{
12747 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12748 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12749
12750 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12751 AssertRCReturn(rc, rc);
12752
12753 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12754 rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
12755 if (rc == VINF_EM_RAW_GUEST_TRAP)
12756 {
12757 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12758 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12759 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12760 AssertRCReturn(rc, rc);
12761
12762 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12763 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12764 }
12765
12766 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
12767 return rc;
12768}
12769
12770
12771/**
12772 * VM-exit exception handler for \#AC (alignment check exception).
12773 */
12774static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12775{
12776 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12777
12778 /*
12779 * Re-inject it. We'll detect any nesting before getting here.
12780 */
12781 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12782 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12783 AssertRCReturn(rc, rc);
12784 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
12785
12786 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12787 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12788 return VINF_SUCCESS;
12789}
12790
12791
12792/**
12793 * VM-exit exception handler for \#DB (Debug exception).
12794 */
12795static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12796{
12797 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12798 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
12799
12800 /*
12801 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
12802 * for processing.
12803 */
12804 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12805
12806 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
12807 uint64_t uDR6 = X86_DR6_INIT_VAL;
12808 uDR6 |= ( pVmxTransient->uExitQualification
12809 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
12810
12811 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12812 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
12813 Log6Func(("rc=%Rrc\n", rc));
12814 if (rc == VINF_EM_RAW_GUEST_TRAP)
12815 {
12816 /*
12817 * The exception was for the guest. Update DR6, DR7.GD and
12818 * IA32_DEBUGCTL.LBR before forwarding it.
12819 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
12820 */
12821 VMMRZCallRing3Disable(pVCpu);
12822 HM_DISABLE_PREEMPT(pVCpu);
12823
12824 pCtx->dr[6] &= ~X86_DR6_B_MASK;
12825 pCtx->dr[6] |= uDR6;
12826 if (CPUMIsGuestDebugStateActive(pVCpu))
12827 ASMSetDR6(pCtx->dr[6]);
12828
12829 HM_RESTORE_PREEMPT();
12830 VMMRZCallRing3Enable(pVCpu);
12831
12832 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12833 AssertRCReturn(rc, rc);
12834
12835 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
12836 pCtx->dr[7] &= ~X86_DR7_GD;
12837
12838 /* Paranoia. */
12839 pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
12840 pCtx->dr[7] |= X86_DR7_RA1_MASK;
12841
12842 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
12843 AssertRCReturn(rc, rc);
12844
12845 /*
12846 * Raise #DB in the guest.
12847 *
12848 * It is important to reflect exactly what the VM-exit gave us (preserving the
12849 * interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
12850 * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
12851 * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
12852 *
12853 * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
12854 * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
12855 */
12856 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12857 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12858 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12859 AssertRCReturn(rc, rc);
12860 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12861 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12862 return VINF_SUCCESS;
12863 }
12864
12865 /*
12866 * Not a guest trap, must be a hypervisor related debug event then.
12867 * Update DR6 in case someone is interested in it.
12868 */
12869 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
12870 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
12871 CPUMSetHyperDR6(pVCpu, uDR6);
12872
12873 return rc;
12874}
12875
12876/**
12877 * VM-exit exception handler for \#GP (General-protection exception).
12878 *
12879 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
12880 */
12881static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12882{
12883 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12884 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
12885
12886 int rc;
12887 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12888 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
12889 { /* likely */ }
12890 else
12891 {
12892#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12893 Assert(pVCpu->hm.s.fUsingDebugLoop);
12894#endif
12895 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
12896 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12897 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12898 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12899 rc |= hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12900 AssertRCReturn(rc, rc);
12901 Log4Func(("Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pCtx->cs.Sel, pCtx->rip,
12902 pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
12903 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12904 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12905 return rc;
12906 }
12907
12908 Assert(CPUMIsGuestInRealModeEx(pCtx));
12909 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
12910
12911 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
12912 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12913 AssertRCReturn(rc, rc);
12914
12915 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12916 uint32_t cbOp = 0;
12917 PVM pVM = pVCpu->CTX_SUFF(pVM);
12918 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12919 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
12920 if (RT_SUCCESS(rc))
12921 {
12922 rc = VINF_SUCCESS;
12923 Assert(cbOp == pDis->cbInstr);
12924 Log4Func(("Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pCtx->cs.Sel, pCtx->rip));
12925 switch (pDis->pCurInstr->uOpcode)
12926 {
12927 case OP_CLI:
12928 {
12929 pCtx->eflags.Bits.u1IF = 0;
12930 pCtx->eflags.Bits.u1RF = 0;
12931 pCtx->rip += pDis->cbInstr;
12932 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12933 if ( !fDbgStepping
12934 && pCtx->eflags.Bits.u1TF)
12935 {
12936 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12937 AssertRCReturn(rc, rc);
12938 }
12939 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
12940 break;
12941 }
12942
12943 case OP_STI:
12944 {
12945 bool fOldIF = pCtx->eflags.Bits.u1IF;
12946 pCtx->eflags.Bits.u1IF = 1;
12947 pCtx->eflags.Bits.u1RF = 0;
12948 pCtx->rip += pDis->cbInstr;
12949 if (!fOldIF)
12950 {
12951 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
12952 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
12953 }
12954 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12955 if ( !fDbgStepping
12956 && pCtx->eflags.Bits.u1TF)
12957 {
12958 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12959 AssertRCReturn(rc, rc);
12960 }
12961 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
12962 break;
12963 }
12964
12965 case OP_HLT:
12966 {
12967 rc = VINF_EM_HALT;
12968 pCtx->rip += pDis->cbInstr;
12969 pCtx->eflags.Bits.u1RF = 0;
12970 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12971 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
12972 break;
12973 }
12974
12975 case OP_POPF:
12976 {
12977 Log4Func(("POPF CS:EIP %04x:%04RX64\n", pCtx->cs.Sel, pCtx->rip));
12978 uint32_t cbParm;
12979 uint32_t uMask;
12980 bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
12981 if (pDis->fPrefix & DISPREFIX_OPSIZE)
12982 {
12983 cbParm = 4;
12984 uMask = 0xffffffff;
12985 }
12986 else
12987 {
12988 cbParm = 2;
12989 uMask = 0xffff;
12990 }
12991
12992 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
12993 RTGCPTR GCPtrStack = 0;
12994 X86EFLAGS Eflags;
12995 Eflags.u32 = 0;
12996 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
12997 &GCPtrStack);
12998 if (RT_SUCCESS(rc))
12999 {
13000 Assert(sizeof(Eflags.u32) >= cbParm);
13001 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13002 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13003 }
13004 if (RT_FAILURE(rc))
13005 {
13006 rc = VERR_EM_INTERPRETER;
13007 break;
13008 }
13009 Log4Func(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pCtx->rsp, uMask, pCtx->rip));
13010 pCtx->eflags.u32 = (pCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13011 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13012 pCtx->esp += cbParm;
13013 pCtx->esp &= uMask;
13014 pCtx->rip += pDis->cbInstr;
13015 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
13016 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13017 POPF restores EFLAGS.TF. */
13018 if ( !fDbgStepping
13019 && fGstStepping)
13020 {
13021 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13022 AssertRCReturn(rc, rc);
13023 }
13024 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13025 break;
13026 }
13027
13028 case OP_PUSHF:
13029 {
13030 uint32_t cbParm;
13031 uint32_t uMask;
13032 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13033 {
13034 cbParm = 4;
13035 uMask = 0xffffffff;
13036 }
13037 else
13038 {
13039 cbParm = 2;
13040 uMask = 0xffff;
13041 }
13042
13043 /* Get the stack pointer & push the contents of eflags onto the stack. */
13044 RTGCPTR GCPtrStack = 0;
13045 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask,
13046 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13047 if (RT_FAILURE(rc))
13048 {
13049 rc = VERR_EM_INTERPRETER;
13050 break;
13051 }
13052 X86EFLAGS Eflags = pCtx->eflags;
13053 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13054 Eflags.Bits.u1RF = 0;
13055 Eflags.Bits.u1VM = 0;
13056
13057 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13058 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13059 {
13060 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13061 rc = VERR_EM_INTERPRETER;
13062 break;
13063 }
13064 Log4Func(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13065 pCtx->esp -= cbParm;
13066 pCtx->esp &= uMask;
13067 pCtx->rip += pDis->cbInstr;
13068 pCtx->eflags.Bits.u1RF = 0;
13069 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
13070 if ( !fDbgStepping
13071 && pCtx->eflags.Bits.u1TF)
13072 {
13073 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13074 AssertRCReturn(rc, rc);
13075 }
13076 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13077 break;
13078 }
13079
13080 case OP_IRET:
13081 {
13082 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13083 * instruction reference. */
13084 RTGCPTR GCPtrStack = 0;
13085 uint32_t uMask = 0xffff;
13086 bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
13087 uint16_t aIretFrame[3];
13088 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13089 {
13090 rc = VERR_EM_INTERPRETER;
13091 break;
13092 }
13093 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13094 &GCPtrStack);
13095 if (RT_SUCCESS(rc))
13096 {
13097 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13098 PGMACCESSORIGIN_HM));
13099 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13100 }
13101 if (RT_FAILURE(rc))
13102 {
13103 rc = VERR_EM_INTERPRETER;
13104 break;
13105 }
13106 pCtx->eip = 0;
13107 pCtx->ip = aIretFrame[0];
13108 pCtx->cs.Sel = aIretFrame[1];
13109 pCtx->cs.ValidSel = aIretFrame[1];
13110 pCtx->cs.u64Base = (uint64_t)pCtx->cs.Sel << 4;
13111 pCtx->eflags.u32 = (pCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13112 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13113 pCtx->sp += sizeof(aIretFrame);
13114 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
13115 | HM_CHANGED_GUEST_CS);
13116 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13117 if ( !fDbgStepping
13118 && fGstStepping)
13119 {
13120 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13121 AssertRCReturn(rc, rc);
13122 }
13123 Log4Func(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pCtx->cs.Sel, pCtx->ip));
13124 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13125 break;
13126 }
13127
13128 case OP_INT:
13129 {
13130 uint16_t uVector = pDis->Param1.uValue & 0xff;
13131 hmR0VmxSetPendingIntN(pVCpu, uVector, pDis->cbInstr);
13132 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13133 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13134 break;
13135 }
13136
13137 case OP_INTO:
13138 {
13139 if (pCtx->eflags.Bits.u1OF)
13140 {
13141 hmR0VmxSetPendingXcptOF(pVCpu, pDis->cbInstr);
13142 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13143 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13144 }
13145 else
13146 {
13147 pCtx->eflags.Bits.u1RF = 0;
13148 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
13149 }
13150 break;
13151 }
13152
13153 default:
13154 {
13155 pCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13156 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pCtx), 0 /* pvFault */,
13157 EMCODETYPE_SUPERVISOR);
13158 rc = VBOXSTRICTRC_VAL(rc2);
13159 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13160 /** @todo We have to set pending-debug exceptions here when the guest is
13161 * single-stepping depending on the instruction that was interpreted. */
13162
13163 /*
13164 * HACK ALERT! Detect mode change and go to ring-3 to properly exit this
13165 * real mode emulation stuff.
13166 */
13167 if ( rc == VINF_SUCCESS
13168 && (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
13169 {
13170 Log4(("hmR0VmxExitXcptGP: mode changed -> VINF_EM_RESCHEDULE\n"));
13171 /** @todo Exit fRealOnV86Active here w/o dropping back to ring-3. */
13172 rc = VINF_EM_RESCHEDULE;
13173 }
13174
13175 Log4Func(("#GP rc=%Rrc\n", rc));
13176 break;
13177 }
13178 }
13179 }
13180 else
13181 rc = VERR_EM_INTERPRETER;
13182
13183 AssertMsg( rc == VINF_SUCCESS
13184 || rc == VERR_EM_INTERPRETER
13185 || rc == VINF_EM_HALT
13186 || rc == VINF_EM_RESCHEDULE
13187 , ("#GP Unexpected rc=%Rrc\n", rc));
13188 return rc;
13189}
13190
13191
13192/**
13193 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13194 * the exception reported in the VMX transient structure back into the VM.
13195 *
13196 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13197 * up-to-date.
13198 */
13199static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13200{
13201 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13202#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13203 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13204 ("uVector=%#x u32XcptBitmap=%#X32\n",
13205 VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13206#endif
13207
13208 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13209 hmR0VmxCheckExitDueToEventDelivery(). */
13210 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13211 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13212 AssertRCReturn(rc, rc);
13213 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13214
13215#ifdef DEBUG_ramshankar
13216 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
13217 uint8_t uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13218 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13219#endif
13220
13221 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13222 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13223 return VINF_SUCCESS;
13224}
13225
13226
13227/**
13228 * VM-exit exception handler for \#PF (Page-fault exception).
13229 */
13230static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13231{
13232 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13233 PVM pVM = pVCpu->CTX_SUFF(pVM);
13234 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13235 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13236 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13237 AssertRCReturn(rc, rc);
13238
13239 if (!pVM->hm.s.fNestedPaging)
13240 { /* likely */ }
13241 else
13242 {
13243#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13244 Assert(pVCpu->hm.s.fUsingDebugLoop);
13245#endif
13246 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13247 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13248 {
13249 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13250 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13251 }
13252 else
13253 {
13254 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13255 hmR0VmxSetPendingXcptDF(pVCpu);
13256 Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13257 }
13258 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13259 return rc;
13260 }
13261
13262 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13263 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13264 if (pVmxTransient->fVectoringPF)
13265 {
13266 Assert(pVCpu->hm.s.Event.fPending);
13267 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13268 }
13269
13270 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13271 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13272 AssertRCReturn(rc, rc);
13273
13274 Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13275 pCtx->cs.Sel, pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
13276
13277 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13278 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx),
13279 (RTGCPTR)pVmxTransient->uExitQualification);
13280
13281 Log4Func(("#PF: rc=%Rrc\n", rc));
13282 if (rc == VINF_SUCCESS)
13283 {
13284 /*
13285 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13286 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13287 */
13288 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13289 TRPMResetTrap(pVCpu);
13290 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13291 return rc;
13292 }
13293
13294 if (rc == VINF_EM_RAW_GUEST_TRAP)
13295 {
13296 if (!pVmxTransient->fVectoringDoublePF)
13297 {
13298 /* It's a guest page fault and needs to be reflected to the guest. */
13299 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13300 TRPMResetTrap(pVCpu);
13301 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13302 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13303 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13304 }
13305 else
13306 {
13307 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13308 TRPMResetTrap(pVCpu);
13309 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13310 hmR0VmxSetPendingXcptDF(pVCpu);
13311 Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13312 }
13313
13314 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13315 return VINF_SUCCESS;
13316 }
13317
13318 TRPMResetTrap(pVCpu);
13319 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13320 return rc;
13321}
13322
13323/** @} */
13324
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette