VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 54384

Last change on this file since 54384 was 54384, checked in by vboxsync, 10 years ago

VMM/HMVMXR0: Fix debug assertion on 32-bit hosts. Host-state is already saved in HMR0Enter() while entering from ring-3/longjmp.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 506.3 KB
Line 
1/* $Id: HMVMXR0.cpp 54384 2015-02-23 15:03:09Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2015 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/x86.h>
23#include <iprt/asm-amd64-x86.h>
24#include <iprt/thread.h>
25
26#include "HMInternal.h"
27#include <VBox/vmm/vm.h>
28#include "HMVMXR0.h"
29#include <VBox/vmm/pdmapi.h>
30#include <VBox/vmm/dbgf.h>
31#include <VBox/vmm/iem.h>
32#include <VBox/vmm/iom.h>
33#include <VBox/vmm/selm.h>
34#include <VBox/vmm/tm.h>
35#include <VBox/vmm/gim.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39# include "dtrace/VBoxVMM.h"
40
41#ifdef DEBUG_ramshankar
42# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
43# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
44# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
45# define HMVMX_ALWAYS_CHECK_GUEST_STATE
46# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
47# define HMVMX_ALWAYS_TRAP_PF
48# define HMVMX_ALWAYS_SWAP_FPU_STATE
49# define HMVMX_ALWAYS_FLUSH_TLB
50# define HMVMX_ALWAYS_SWAP_EFER
51#endif
52
53
54/*******************************************************************************
55* Defined Constants And Macros *
56*******************************************************************************/
57#if defined(RT_ARCH_AMD64)
58# define HMVMX_IS_64BIT_HOST_MODE() (true)
59typedef RTHCUINTREG HMVMXHCUINTREG;
60#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
61extern "C" uint32_t g_fVMXIs64bitHost;
62# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
63typedef uint64_t HMVMXHCUINTREG;
64#else
65# define HMVMX_IS_64BIT_HOST_MODE() (false)
66typedef RTHCUINTREG HMVMXHCUINTREG;
67#endif
68
69/** Use the function table. */
70#define HMVMX_USE_FUNCTION_TABLE
71
72/** Determine which tagged-TLB flush handler to use. */
73#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
74#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
75#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
76#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
77
78/** @name Updated-guest-state flags.
79 * @{ */
80#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
81#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
82#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
83#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
84#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
85#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
86#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
87#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
88#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
89#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
90#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
91#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
92#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
93#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
94#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
95#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
96#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
97#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
98#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
99#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
100#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
101 | HMVMX_UPDATED_GUEST_RSP \
102 | HMVMX_UPDATED_GUEST_RFLAGS \
103 | HMVMX_UPDATED_GUEST_CR0 \
104 | HMVMX_UPDATED_GUEST_CR3 \
105 | HMVMX_UPDATED_GUEST_CR4 \
106 | HMVMX_UPDATED_GUEST_GDTR \
107 | HMVMX_UPDATED_GUEST_IDTR \
108 | HMVMX_UPDATED_GUEST_LDTR \
109 | HMVMX_UPDATED_GUEST_TR \
110 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
111 | HMVMX_UPDATED_GUEST_DEBUG \
112 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
113 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
114 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
115 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
116 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
117 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
118 | HMVMX_UPDATED_GUEST_INTR_STATE \
119 | HMVMX_UPDATED_GUEST_APIC_STATE)
120/** @} */
121
122/** @name
123 * Flags to skip redundant reads of some common VMCS fields that are not part of
124 * the guest-CPU state but are in the transient structure.
125 */
126#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
127#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
128#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
129#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
130#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
131#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
132#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
133/** @} */
134
135/** @name
136 * States of the VMCS.
137 *
138 * This does not reflect all possible VMCS states but currently only those
139 * needed for maintaining the VMCS consistently even when thread-context hooks
140 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
141 */
142#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
143#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
144#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
145/** @} */
146
147/**
148 * Exception bitmap mask for real-mode guests (real-on-v86).
149 *
150 * We need to intercept all exceptions manually except:
151 * - #NM, #MF handled in hmR0VmxLoadSharedCR0().
152 * - #DB handled in hmR0VmxLoadSharedDebugState().
153 * - #PF need not be intercepted even in real-mode if we have Nested Paging
154 * support.
155 */
156#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
157 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
158 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
159 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
160 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
161 /* RT_BIT(X86_XCPT_MF) */ | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
162 | RT_BIT(X86_XCPT_XF))
163
164/**
165 * Exception bitmap mask for all contributory exceptions.
166 *
167 * Page fault is deliberately excluded here as it's conditional as to whether
168 * it's contributory or benign. Page faults are handled separately.
169 */
170#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
171 | RT_BIT(X86_XCPT_DE))
172
173/** Maximum VM-instruction error number. */
174#define HMVMX_INSTR_ERROR_MAX 28
175
176/** Profiling macro. */
177#ifdef HM_PROFILE_EXIT_DISPATCH
178# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
179# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
180#else
181# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
182# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
183#endif
184
185/** Assert that preemption is disabled or covered by thread-context hooks. */
186#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
187 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
188
189/** Assert that we haven't migrated CPUs when thread-context hooks are not
190 * used. */
191#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
192 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
193 ("Illegal migration! Entered on CPU %u Current %u\n", \
194 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
195
196/** Helper macro for VM-exit handlers called unexpectedly. */
197#define HMVMX_RETURN_UNEXPECTED_EXIT() \
198 do { \
199 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
200 return VERR_VMX_UNEXPECTED_EXIT; \
201 } while (0)
202
203
204/*******************************************************************************
205* Structures and Typedefs *
206*******************************************************************************/
207/**
208 * VMX transient state.
209 *
210 * A state structure for holding miscellaneous information across
211 * VMX non-root operation and restored after the transition.
212 */
213typedef struct VMXTRANSIENT
214{
215 /** The host's rflags/eflags. */
216 RTCCUINTREG uEflags;
217#if HC_ARCH_BITS == 32
218 uint32_t u32Alignment0;
219#endif
220 /** The guest's TPR value used for TPR shadowing. */
221 uint8_t u8GuestTpr;
222 /** Alignment. */
223 uint8_t abAlignment0[7];
224
225 /** The basic VM-exit reason. */
226 uint16_t uExitReason;
227 /** Alignment. */
228 uint16_t u16Alignment0;
229 /** The VM-exit interruption error code. */
230 uint32_t uExitIntErrorCode;
231 /** The VM-exit exit code qualification. */
232 uint64_t uExitQualification;
233
234 /** The VM-exit interruption-information field. */
235 uint32_t uExitIntInfo;
236 /** The VM-exit instruction-length field. */
237 uint32_t cbInstr;
238 /** The VM-exit instruction-information field. */
239 union
240 {
241 /** Plain unsigned int representation. */
242 uint32_t u;
243 /** INS and OUTS information. */
244 struct
245 {
246 uint32_t u6Reserved0 : 7;
247 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
248 uint32_t u3AddrSize : 3;
249 uint32_t u5Reserved1 : 5;
250 /** The segment register (X86_SREG_XXX). */
251 uint32_t iSegReg : 3;
252 uint32_t uReserved2 : 14;
253 } StrIo;
254 } ExitInstrInfo;
255 /** Whether the VM-entry failed or not. */
256 bool fVMEntryFailed;
257 /** Alignment. */
258 uint8_t abAlignment1[3];
259
260 /** The VM-entry interruption-information field. */
261 uint32_t uEntryIntInfo;
262 /** The VM-entry exception error code field. */
263 uint32_t uEntryXcptErrorCode;
264 /** The VM-entry instruction length field. */
265 uint32_t cbEntryInstr;
266
267 /** IDT-vectoring information field. */
268 uint32_t uIdtVectoringInfo;
269 /** IDT-vectoring error code. */
270 uint32_t uIdtVectoringErrorCode;
271
272 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
273 uint32_t fVmcsFieldsRead;
274
275 /** Whether the guest FPU was active at the time of VM-exit. */
276 bool fWasGuestFPUStateActive;
277 /** Whether the guest debug state was active at the time of VM-exit. */
278 bool fWasGuestDebugStateActive;
279 /** Whether the hyper debug state was active at the time of VM-exit. */
280 bool fWasHyperDebugStateActive;
281 /** Whether TSC-offsetting should be setup before VM-entry. */
282 bool fUpdateTscOffsettingAndPreemptTimer;
283 /** Whether the VM-exit was caused by a page-fault during delivery of a
284 * contributory exception or a page-fault. */
285 bool fVectoringDoublePF;
286 /** Whether the VM-exit was caused by a page-fault during delivery of an
287 * external interrupt or NMI. */
288 bool fVectoringPF;
289} VMXTRANSIENT;
290AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
291AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
292AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
293AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
294AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
295/** Pointer to VMX transient state. */
296typedef VMXTRANSIENT *PVMXTRANSIENT;
297
298
299/**
300 * MSR-bitmap read permissions.
301 */
302typedef enum VMXMSREXITREAD
303{
304 /** Reading this MSR causes a VM-exit. */
305 VMXMSREXIT_INTERCEPT_READ = 0xb,
306 /** Reading this MSR does not cause a VM-exit. */
307 VMXMSREXIT_PASSTHRU_READ
308} VMXMSREXITREAD;
309/** Pointer to MSR-bitmap read permissions. */
310typedef VMXMSREXITREAD* PVMXMSREXITREAD;
311
312/**
313 * MSR-bitmap write permissions.
314 */
315typedef enum VMXMSREXITWRITE
316{
317 /** Writing to this MSR causes a VM-exit. */
318 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
319 /** Writing to this MSR does not cause a VM-exit. */
320 VMXMSREXIT_PASSTHRU_WRITE
321} VMXMSREXITWRITE;
322/** Pointer to MSR-bitmap write permissions. */
323typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
324
325
326/**
327 * VMX VM-exit handler.
328 *
329 * @returns VBox status code.
330 * @param pVCpu Pointer to the VMCPU.
331 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
332 * out-of-sync. Make sure to update the required
333 * fields before using them.
334 * @param pVmxTransient Pointer to the VMX-transient structure.
335 */
336#ifndef HMVMX_USE_FUNCTION_TABLE
337typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
338#else
339typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
340/** Pointer to VM-exit handler. */
341typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
342#endif
343
344
345/*******************************************************************************
346* Internal Functions *
347*******************************************************************************/
348static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
349static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
350static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
351 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
352 bool fStepping, uint32_t *puIntState);
353#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
354static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
355#endif
356#ifndef HMVMX_USE_FUNCTION_TABLE
357DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
358# define HMVMX_EXIT_DECL static int
359#else
360# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
361#endif
362DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitStep(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
363 uint32_t uExitReason, uint16_t uCsStart, uint64_t uRipStart);
364
365/** @name VM-exit handlers.
366 * @{
367 */
368static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
369static FNVMXEXITHANDLER hmR0VmxExitExtInt;
370static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
371static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
372static FNVMXEXITHANDLER hmR0VmxExitSipi;
373static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
374static FNVMXEXITHANDLER hmR0VmxExitSmi;
375static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
376static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
377static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
378static FNVMXEXITHANDLER hmR0VmxExitCpuid;
379static FNVMXEXITHANDLER hmR0VmxExitGetsec;
380static FNVMXEXITHANDLER hmR0VmxExitHlt;
381static FNVMXEXITHANDLER hmR0VmxExitInvd;
382static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
383static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
384static FNVMXEXITHANDLER hmR0VmxExitVmcall;
385static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
386static FNVMXEXITHANDLER hmR0VmxExitRsm;
387static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
388static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
389static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
390static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
391static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
392static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
393static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
394static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
395static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
396static FNVMXEXITHANDLER hmR0VmxExitMwait;
397static FNVMXEXITHANDLER hmR0VmxExitMtf;
398static FNVMXEXITHANDLER hmR0VmxExitMonitor;
399static FNVMXEXITHANDLER hmR0VmxExitPause;
400static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
401static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
402static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
403static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
404static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
405static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
406static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
407static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
408static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
409static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
410static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
411static FNVMXEXITHANDLER hmR0VmxExitRdrand;
412static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
413/** @} */
414
415static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
416static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
417static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
418static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
419static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
420static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
421#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
422static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
423#endif
424static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
425
426/*******************************************************************************
427* Global Variables *
428*******************************************************************************/
429#ifdef HMVMX_USE_FUNCTION_TABLE
430
431/**
432 * VMX_EXIT dispatch table.
433 */
434static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
435{
436 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
437 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
438 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
439 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
440 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
441 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
442 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
443 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
444 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
445 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
446 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
447 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
448 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
449 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
450 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
451 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
452 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
453 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
454 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
455 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
456 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
457 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
458 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
459 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
460 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
461 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
462 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
463 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
464 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
465 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
466 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
467 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
468 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
469 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
470 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
471 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
472 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
473 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
474 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
475 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
476 /* 40 UNDEFINED */ hmR0VmxExitPause,
477 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
478 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
479 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
480 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
481 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
482 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
483 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
484 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
485 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
486 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
487 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
488 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
489 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
490 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
491 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
492 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
493 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
494 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
495 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
496};
497#endif /* HMVMX_USE_FUNCTION_TABLE */
498
499#ifdef VBOX_STRICT
500static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
501{
502 /* 0 */ "(Not Used)",
503 /* 1 */ "VMCALL executed in VMX root operation.",
504 /* 2 */ "VMCLEAR with invalid physical address.",
505 /* 3 */ "VMCLEAR with VMXON pointer.",
506 /* 4 */ "VMLAUNCH with non-clear VMCS.",
507 /* 5 */ "VMRESUME with non-launched VMCS.",
508 /* 6 */ "VMRESUME after VMXOFF",
509 /* 7 */ "VM-entry with invalid control fields.",
510 /* 8 */ "VM-entry with invalid host state fields.",
511 /* 9 */ "VMPTRLD with invalid physical address.",
512 /* 10 */ "VMPTRLD with VMXON pointer.",
513 /* 11 */ "VMPTRLD with incorrect revision identifier.",
514 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
515 /* 13 */ "VMWRITE to read-only VMCS component.",
516 /* 14 */ "(Not Used)",
517 /* 15 */ "VMXON executed in VMX root operation.",
518 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
519 /* 17 */ "VM-entry with non-launched executing VMCS.",
520 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
521 /* 19 */ "VMCALL with non-clear VMCS.",
522 /* 20 */ "VMCALL with invalid VM-exit control fields.",
523 /* 21 */ "(Not Used)",
524 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
525 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
526 /* 24 */ "VMCALL with invalid SMM-monitor features.",
527 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
528 /* 26 */ "VM-entry with events blocked by MOV SS.",
529 /* 27 */ "(Not Used)",
530 /* 28 */ "Invalid operand to INVEPT/INVVPID."
531};
532#endif /* VBOX_STRICT */
533
534
535
536/**
537 * Updates the VM's last error record. If there was a VMX instruction error,
538 * reads the error data from the VMCS and updates VCPU's last error record as
539 * well.
540 *
541 * @param pVM Pointer to the VM.
542 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
543 * VERR_VMX_UNABLE_TO_START_VM or
544 * VERR_VMX_INVALID_VMCS_FIELD).
545 * @param rc The error code.
546 */
547static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
548{
549 AssertPtr(pVM);
550 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
551 || rc == VERR_VMX_UNABLE_TO_START_VM)
552 {
553 AssertPtrReturnVoid(pVCpu);
554 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
555 }
556 pVM->hm.s.lLastError = rc;
557}
558
559
560/**
561 * Reads the VM-entry interruption-information field from the VMCS into the VMX
562 * transient structure.
563 *
564 * @returns VBox status code.
565 * @param pVmxTransient Pointer to the VMX transient structure.
566 *
567 * @remarks No-long-jump zone!!!
568 */
569DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
570{
571 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
572 AssertRCReturn(rc, rc);
573 return VINF_SUCCESS;
574}
575
576
577/**
578 * Reads the VM-entry exception error code field from the VMCS into
579 * the VMX transient structure.
580 *
581 * @returns VBox status code.
582 * @param pVmxTransient Pointer to the VMX transient structure.
583 *
584 * @remarks No-long-jump zone!!!
585 */
586DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
587{
588 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
589 AssertRCReturn(rc, rc);
590 return VINF_SUCCESS;
591}
592
593
594/**
595 * Reads the VM-entry exception error code field from the VMCS into
596 * the VMX transient structure.
597 *
598 * @returns VBox status code.
599 * @param pVmxTransient Pointer to the VMX transient structure.
600 *
601 * @remarks No-long-jump zone!!!
602 */
603DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
604{
605 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
606 AssertRCReturn(rc, rc);
607 return VINF_SUCCESS;
608}
609
610
611/**
612 * Reads the VM-exit interruption-information field from the VMCS into the VMX
613 * transient structure.
614 *
615 * @returns VBox status code.
616 * @param pVmxTransient Pointer to the VMX transient structure.
617 */
618DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
619{
620 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
621 {
622 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
623 AssertRCReturn(rc, rc);
624 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
625 }
626 return VINF_SUCCESS;
627}
628
629
630/**
631 * Reads the VM-exit interruption error code from the VMCS into the VMX
632 * transient structure.
633 *
634 * @returns VBox status code.
635 * @param pVmxTransient Pointer to the VMX transient structure.
636 */
637DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
638{
639 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
640 {
641 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
642 AssertRCReturn(rc, rc);
643 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
644 }
645 return VINF_SUCCESS;
646}
647
648
649/**
650 * Reads the VM-exit instruction length field from the VMCS into the VMX
651 * transient structure.
652 *
653 * @returns VBox status code.
654 * @param pVCpu Pointer to the VMCPU.
655 * @param pVmxTransient Pointer to the VMX transient structure.
656 */
657DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
658{
659 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
660 {
661 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
662 AssertRCReturn(rc, rc);
663 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
664 }
665 return VINF_SUCCESS;
666}
667
668
669/**
670 * Reads the VM-exit instruction-information field from the VMCS into
671 * the VMX transient structure.
672 *
673 * @returns VBox status code.
674 * @param pVmxTransient Pointer to the VMX transient structure.
675 */
676DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
677{
678 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
679 {
680 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
681 AssertRCReturn(rc, rc);
682 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
683 }
684 return VINF_SUCCESS;
685}
686
687
688/**
689 * Reads the exit code qualification from the VMCS into the VMX transient
690 * structure.
691 *
692 * @returns VBox status code.
693 * @param pVCpu Pointer to the VMCPU (required for the VMCS cache
694 * case).
695 * @param pVmxTransient Pointer to the VMX transient structure.
696 */
697DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
698{
699 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
700 {
701 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
702 AssertRCReturn(rc, rc);
703 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
704 }
705 return VINF_SUCCESS;
706}
707
708
709/**
710 * Reads the IDT-vectoring information field from the VMCS into the VMX
711 * transient structure.
712 *
713 * @returns VBox status code.
714 * @param pVmxTransient Pointer to the VMX transient structure.
715 *
716 * @remarks No-long-jump zone!!!
717 */
718DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
719{
720 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
721 {
722 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
723 AssertRCReturn(rc, rc);
724 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
725 }
726 return VINF_SUCCESS;
727}
728
729
730/**
731 * Reads the IDT-vectoring error code from the VMCS into the VMX
732 * transient structure.
733 *
734 * @returns VBox status code.
735 * @param pVmxTransient Pointer to the VMX transient structure.
736 */
737DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
738{
739 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
740 {
741 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
742 AssertRCReturn(rc, rc);
743 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
744 }
745 return VINF_SUCCESS;
746}
747
748
749/**
750 * Enters VMX root mode operation on the current CPU.
751 *
752 * @returns VBox status code.
753 * @param pVM Pointer to the VM (optional, can be NULL, after
754 * a resume).
755 * @param HCPhysCpuPage Physical address of the VMXON region.
756 * @param pvCpuPage Pointer to the VMXON region.
757 */
758static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
759{
760 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
761 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
762 Assert(pvCpuPage);
763 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
764
765 if (pVM)
766 {
767 /* Write the VMCS revision dword to the VMXON region. */
768 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
769 }
770
771 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
772 RTCCUINTREG uEflags = ASMIntDisableFlags();
773
774 /* Enable the VMX bit in CR4 if necessary. */
775 RTCCUINTREG uCr4 = ASMGetCR4();
776 if (!(uCr4 & X86_CR4_VMXE))
777 ASMSetCR4(uCr4 | X86_CR4_VMXE);
778
779 /* Enter VMX root mode. */
780 int rc = VMXEnable(HCPhysCpuPage);
781 if (RT_FAILURE(rc))
782 ASMSetCR4(uCr4);
783
784 /* Restore interrupts. */
785 ASMSetFlags(uEflags);
786 return rc;
787}
788
789
790/**
791 * Exits VMX root mode operation on the current CPU.
792 *
793 * @returns VBox status code.
794 */
795static int hmR0VmxLeaveRootMode(void)
796{
797 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
798
799 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
800 RTCCUINTREG uEflags = ASMIntDisableFlags();
801
802 /* If we're for some reason not in VMX root mode, then don't leave it. */
803 RTCCUINTREG uHostCR4 = ASMGetCR4();
804
805 int rc;
806 if (uHostCR4 & X86_CR4_VMXE)
807 {
808 /* Exit VMX root mode and clear the VMX bit in CR4. */
809 VMXDisable();
810 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
811 rc = VINF_SUCCESS;
812 }
813 else
814 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
815
816 /* Restore interrupts. */
817 ASMSetFlags(uEflags);
818 return rc;
819}
820
821
822/**
823 * Allocates and maps one physically contiguous page. The allocated page is
824 * zero'd out. (Used by various VT-x structures).
825 *
826 * @returns IPRT status code.
827 * @param pMemObj Pointer to the ring-0 memory object.
828 * @param ppVirt Where to store the virtual address of the
829 * allocation.
830 * @param pPhys Where to store the physical address of the
831 * allocation.
832 */
833DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
834{
835 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
836 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
837 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
838
839 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
840 if (RT_FAILURE(rc))
841 return rc;
842 *ppVirt = RTR0MemObjAddress(*pMemObj);
843 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
844 ASMMemZero32(*ppVirt, PAGE_SIZE);
845 return VINF_SUCCESS;
846}
847
848
849/**
850 * Frees and unmaps an allocated physical page.
851 *
852 * @param pMemObj Pointer to the ring-0 memory object.
853 * @param ppVirt Where to re-initialize the virtual address of
854 * allocation as 0.
855 * @param pHCPhys Where to re-initialize the physical address of the
856 * allocation as 0.
857 */
858DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
859{
860 AssertPtr(pMemObj);
861 AssertPtr(ppVirt);
862 AssertPtr(pHCPhys);
863 if (*pMemObj != NIL_RTR0MEMOBJ)
864 {
865 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
866 AssertRC(rc);
867 *pMemObj = NIL_RTR0MEMOBJ;
868 *ppVirt = 0;
869 *pHCPhys = 0;
870 }
871}
872
873
874/**
875 * Worker function to free VT-x related structures.
876 *
877 * @returns IPRT status code.
878 * @param pVM Pointer to the VM.
879 */
880static void hmR0VmxStructsFree(PVM pVM)
881{
882 for (VMCPUID i = 0; i < pVM->cCpus; i++)
883 {
884 PVMCPU pVCpu = &pVM->aCpus[i];
885 AssertPtr(pVCpu);
886
887 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
888 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
889
890 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
891 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
892
893 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
894 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
895 }
896
897 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
898#ifdef VBOX_WITH_CRASHDUMP_MAGIC
899 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
900#endif
901}
902
903
904/**
905 * Worker function to allocate VT-x related VM structures.
906 *
907 * @returns IPRT status code.
908 * @param pVM Pointer to the VM.
909 */
910static int hmR0VmxStructsAlloc(PVM pVM)
911{
912 /*
913 * Initialize members up-front so we can cleanup properly on allocation failure.
914 */
915#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
916 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
917 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
918 pVM->hm.s.vmx.HCPhys##a_Name = 0;
919
920#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
921 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
922 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
923 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
924
925#ifdef VBOX_WITH_CRASHDUMP_MAGIC
926 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
927#endif
928 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
929
930 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
931 for (VMCPUID i = 0; i < pVM->cCpus; i++)
932 {
933 PVMCPU pVCpu = &pVM->aCpus[i];
934 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
935 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
936 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
937 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
938 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
939 }
940#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
941#undef VMXLOCAL_INIT_VM_MEMOBJ
942
943 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
944 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
945 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
946 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
947
948 /*
949 * Allocate all the VT-x structures.
950 */
951 int rc = VINF_SUCCESS;
952#ifdef VBOX_WITH_CRASHDUMP_MAGIC
953 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
954 if (RT_FAILURE(rc))
955 goto cleanup;
956 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
957 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
958#endif
959
960 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
961 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
962 {
963 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
964 &pVM->hm.s.vmx.HCPhysApicAccess);
965 if (RT_FAILURE(rc))
966 goto cleanup;
967 }
968
969 /*
970 * Initialize per-VCPU VT-x structures.
971 */
972 for (VMCPUID i = 0; i < pVM->cCpus; i++)
973 {
974 PVMCPU pVCpu = &pVM->aCpus[i];
975 AssertPtr(pVCpu);
976
977 /* Allocate the VM control structure (VMCS). */
978 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
979 if (RT_FAILURE(rc))
980 goto cleanup;
981
982 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
983 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
984 {
985 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
986 &pVCpu->hm.s.vmx.HCPhysVirtApic);
987 if (RT_FAILURE(rc))
988 goto cleanup;
989 }
990
991 /*
992 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
993 * transparent accesses of specific MSRs.
994 *
995 * If the condition for enabling MSR bitmaps changes here, don't forget to
996 * update HMIsMsrBitmapsAvailable().
997 */
998 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
999 {
1000 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1001 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1002 if (RT_FAILURE(rc))
1003 goto cleanup;
1004 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1005 }
1006
1007 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1008 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1009 if (RT_FAILURE(rc))
1010 goto cleanup;
1011
1012 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1013 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1014 if (RT_FAILURE(rc))
1015 goto cleanup;
1016 }
1017
1018 return VINF_SUCCESS;
1019
1020cleanup:
1021 hmR0VmxStructsFree(pVM);
1022 return rc;
1023}
1024
1025
1026/**
1027 * Does global VT-x initialization (called during module initialization).
1028 *
1029 * @returns VBox status code.
1030 */
1031VMMR0DECL(int) VMXR0GlobalInit(void)
1032{
1033#ifdef HMVMX_USE_FUNCTION_TABLE
1034 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1035# ifdef VBOX_STRICT
1036 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1037 Assert(g_apfnVMExitHandlers[i]);
1038# endif
1039#endif
1040 return VINF_SUCCESS;
1041}
1042
1043
1044/**
1045 * Does global VT-x termination (called during module termination).
1046 */
1047VMMR0DECL(void) VMXR0GlobalTerm()
1048{
1049 /* Nothing to do currently. */
1050}
1051
1052
1053/**
1054 * Sets up and activates VT-x on the current CPU.
1055 *
1056 * @returns VBox status code.
1057 * @param pCpu Pointer to the global CPU info struct.
1058 * @param pVM Pointer to the VM (can be NULL after a host resume
1059 * operation).
1060 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1061 * fEnabledByHost is true).
1062 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1063 * @a fEnabledByHost is true).
1064 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1065 * enable VT-x on the host.
1066 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1067 */
1068VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1069 void *pvMsrs)
1070{
1071 Assert(pCpu);
1072 Assert(pvMsrs);
1073 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1074
1075 /* Enable VT-x if it's not already enabled by the host. */
1076 if (!fEnabledByHost)
1077 {
1078 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1079 if (RT_FAILURE(rc))
1080 return rc;
1081 }
1082
1083 /*
1084 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1085 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1086 */
1087 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1088 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1089 {
1090 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1091 pCpu->fFlushAsidBeforeUse = false;
1092 }
1093 else
1094 pCpu->fFlushAsidBeforeUse = true;
1095
1096 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1097 ++pCpu->cTlbFlushes;
1098
1099 return VINF_SUCCESS;
1100}
1101
1102
1103/**
1104 * Deactivates VT-x on the current CPU.
1105 *
1106 * @returns VBox status code.
1107 * @param pCpu Pointer to the global CPU info struct.
1108 * @param pvCpuPage Pointer to the VMXON region.
1109 * @param HCPhysCpuPage Physical address of the VMXON region.
1110 *
1111 * @remarks This function should never be called when SUPR0EnableVTx() or
1112 * similar was used to enable VT-x on the host.
1113 */
1114VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1115{
1116 NOREF(pCpu);
1117 NOREF(pvCpuPage);
1118 NOREF(HCPhysCpuPage);
1119
1120 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1121 return hmR0VmxLeaveRootMode();
1122}
1123
1124
1125/**
1126 * Sets the permission bits for the specified MSR in the MSR bitmap.
1127 *
1128 * @param pVCpu Pointer to the VMCPU.
1129 * @param uMSR The MSR value.
1130 * @param enmRead Whether reading this MSR causes a VM-exit.
1131 * @param enmWrite Whether writing this MSR causes a VM-exit.
1132 */
1133static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1134{
1135 int32_t iBit;
1136 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1137
1138 /*
1139 * Layout:
1140 * 0x000 - 0x3ff - Low MSR read bits
1141 * 0x400 - 0x7ff - High MSR read bits
1142 * 0x800 - 0xbff - Low MSR write bits
1143 * 0xc00 - 0xfff - High MSR write bits
1144 */
1145 if (uMsr <= 0x00001FFF)
1146 iBit = uMsr;
1147 else if ( uMsr >= 0xC0000000
1148 && uMsr <= 0xC0001FFF)
1149 {
1150 iBit = (uMsr - 0xC0000000);
1151 pbMsrBitmap += 0x400;
1152 }
1153 else
1154 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1155
1156 Assert(iBit <= 0x1fff);
1157 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1158 ASMBitSet(pbMsrBitmap, iBit);
1159 else
1160 ASMBitClear(pbMsrBitmap, iBit);
1161
1162 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1163 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1164 else
1165 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1166}
1167
1168
1169#ifdef VBOX_STRICT
1170/**
1171 * Gets the permission bits for the specified MSR in the MSR bitmap.
1172 *
1173 * @returns VBox status code.
1174 * @retval VINF_SUCCESS if the specified MSR is found.
1175 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1176 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1177 *
1178 * @param pVCpu Pointer to the VMCPU.
1179 * @param uMsr The MSR.
1180 * @param penmRead Where to store the read permissions.
1181 * @param penmWrite Where to store the write permissions.
1182 */
1183static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1184{
1185 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1186 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1187 int32_t iBit;
1188 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1189
1190 /* See hmR0VmxSetMsrPermission() for the layout. */
1191 if (uMsr <= 0x00001FFF)
1192 iBit = uMsr;
1193 else if ( uMsr >= 0xC0000000
1194 && uMsr <= 0xC0001FFF)
1195 {
1196 iBit = (uMsr - 0xC0000000);
1197 pbMsrBitmap += 0x400;
1198 }
1199 else
1200 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1201
1202 Assert(iBit <= 0x1fff);
1203 if (ASMBitTest(pbMsrBitmap, iBit))
1204 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1205 else
1206 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1207
1208 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1209 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1210 else
1211 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1212 return VINF_SUCCESS;
1213}
1214#endif /* VBOX_STRICT */
1215
1216
1217/**
1218 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1219 * area.
1220 *
1221 * @returns VBox status code.
1222 * @param pVCpu Pointer to the VMCPU.
1223 * @param cMsrs The number of MSRs.
1224 */
1225DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1226{
1227 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1228 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1229 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1230 {
1231 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1232 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1233 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1234 }
1235
1236 /* Update number of guest MSRs to load/store across the world-switch. */
1237 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1238 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1239
1240 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1241 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1242
1243 /* Update the VCPU's copy of the MSR count. */
1244 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1245
1246 return VINF_SUCCESS;
1247}
1248
1249
1250/**
1251 * Adds a new (or updates the value of an existing) guest/host MSR
1252 * pair to be swapped during the world-switch as part of the
1253 * auto-load/store MSR area in the VMCS.
1254 *
1255 * @returns true if the MSR was added -and- its value was updated, false
1256 * otherwise.
1257 * @param pVCpu Pointer to the VMCPU.
1258 * @param uMsr The MSR.
1259 * @param uGuestMsr Value of the guest MSR.
1260 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1261 * necessary.
1262 */
1263static bool hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1264{
1265 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1266 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1267 uint32_t i;
1268 for (i = 0; i < cMsrs; i++)
1269 {
1270 if (pGuestMsr->u32Msr == uMsr)
1271 break;
1272 pGuestMsr++;
1273 }
1274
1275 bool fAdded = false;
1276 if (i == cMsrs)
1277 {
1278 ++cMsrs;
1279 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1280 AssertRC(rc);
1281
1282 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1283 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1284 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1285
1286 fAdded = true;
1287 }
1288
1289 /* Update the MSR values in the auto-load/store MSR area. */
1290 pGuestMsr->u32Msr = uMsr;
1291 pGuestMsr->u64Value = uGuestMsrValue;
1292
1293 /* Create/update the MSR slot in the host MSR area. */
1294 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1295 pHostMsr += i;
1296 pHostMsr->u32Msr = uMsr;
1297
1298 /*
1299 * Update the host MSR only when requested by the caller AND when we're
1300 * adding it to the auto-load/store area. Otherwise, it would have been
1301 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1302 */
1303 bool fUpdatedMsrValue = false;
1304 if ( fAdded
1305 && fUpdateHostMsr)
1306 {
1307 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1308 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1309 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1310 fUpdatedMsrValue = true;
1311 }
1312
1313 return fUpdatedMsrValue;
1314}
1315
1316
1317/**
1318 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1319 * auto-load/store MSR area in the VMCS.
1320 *
1321 * @returns VBox status code.
1322 * @param pVCpu Pointer to the VMCPU.
1323 * @param uMsr The MSR.
1324 */
1325static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1326{
1327 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1328 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1329 for (uint32_t i = 0; i < cMsrs; i++)
1330 {
1331 /* Find the MSR. */
1332 if (pGuestMsr->u32Msr == uMsr)
1333 {
1334 /* If it's the last MSR, simply reduce the count. */
1335 if (i == cMsrs - 1)
1336 {
1337 --cMsrs;
1338 break;
1339 }
1340
1341 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1342 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1343 pLastGuestMsr += cMsrs - 1;
1344 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1345 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1346
1347 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1348 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1349 pLastHostMsr += cMsrs - 1;
1350 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1351 pHostMsr->u64Value = pLastHostMsr->u64Value;
1352 --cMsrs;
1353 break;
1354 }
1355 pGuestMsr++;
1356 }
1357
1358 /* Update the VMCS if the count changed (meaning the MSR was found). */
1359 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1360 {
1361 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1362 AssertRCReturn(rc, rc);
1363
1364 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1365 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1366 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1367
1368 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1369 return VINF_SUCCESS;
1370 }
1371
1372 return VERR_NOT_FOUND;
1373}
1374
1375
1376/**
1377 * Checks if the specified guest MSR is part of the auto-load/store area in
1378 * the VMCS.
1379 *
1380 * @returns true if found, false otherwise.
1381 * @param pVCpu Pointer to the VMCPU.
1382 * @param uMsr The MSR to find.
1383 */
1384static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1385{
1386 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1387 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1388
1389 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1390 {
1391 if (pGuestMsr->u32Msr == uMsr)
1392 return true;
1393 }
1394 return false;
1395}
1396
1397
1398/**
1399 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1400 *
1401 * @param pVCpu Pointer to the VMCPU.
1402 *
1403 * @remarks No-long-jump zone!!!
1404 */
1405static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1406{
1407 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1408 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1409 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1410 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1411
1412 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1413 {
1414 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1415
1416 /*
1417 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1418 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1419 */
1420 if (pHostMsr->u32Msr == MSR_K6_EFER)
1421 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1422 else
1423 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1424 }
1425
1426 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1427}
1428
1429
1430#if HC_ARCH_BITS == 64
1431/**
1432 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1433 * perform lazy restoration of the host MSRs while leaving VT-x.
1434 *
1435 * @param pVCpu Pointer to the VMCPU.
1436 *
1437 * @remarks No-long-jump zone!!!
1438 */
1439static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1440{
1441 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1442
1443 /*
1444 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1445 */
1446 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1447 {
1448 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1449 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1450 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1451 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1452 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1453 }
1454}
1455
1456
1457/**
1458 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1459 * lazily while leaving VT-x.
1460 *
1461 * @returns true if it does, false otherwise.
1462 * @param pVCpu Pointer to the VMCPU.
1463 * @param uMsr The MSR to check.
1464 */
1465static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1466{
1467 NOREF(pVCpu);
1468 switch (uMsr)
1469 {
1470 case MSR_K8_LSTAR:
1471 case MSR_K6_STAR:
1472 case MSR_K8_SF_MASK:
1473 case MSR_K8_KERNEL_GS_BASE:
1474 return true;
1475 }
1476 return false;
1477}
1478
1479
1480/**
1481 * Saves a set of guest MSRs back into the guest-CPU context.
1482 *
1483 * @param pVCpu Pointer to the VMCPU.
1484 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1485 * out-of-sync. Make sure to update the required fields
1486 * before using them.
1487 *
1488 * @remarks No-long-jump zone!!!
1489 */
1490static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1491{
1492 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1493 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1494
1495 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1496 {
1497 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1498 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1499 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1500 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1501 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1502 }
1503}
1504
1505
1506/**
1507 * Loads a set of guests MSRs to allow read/passthru to the guest.
1508 *
1509 * The name of this function is slightly confusing. This function does NOT
1510 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1511 * common prefix for functions dealing with "lazy restoration" of the shared
1512 * MSRs.
1513 *
1514 * @param pVCpu Pointer to the VMCPU.
1515 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1516 * out-of-sync. Make sure to update the required fields
1517 * before using them.
1518 *
1519 * @remarks No-long-jump zone!!!
1520 */
1521static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1522{
1523 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1524 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1525
1526#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1527 do { \
1528 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1529 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1530 else \
1531 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1532 } while (0)
1533
1534 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1535 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1536 {
1537 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1538 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1539 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1540 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1541 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1542 }
1543 else
1544 {
1545 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1546 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1547 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1548 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1549 }
1550
1551#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1552}
1553
1554
1555/**
1556 * Performs lazy restoration of the set of host MSRs if they were previously
1557 * loaded with guest MSR values.
1558 *
1559 * @param pVCpu Pointer to the VMCPU.
1560 *
1561 * @remarks No-long-jump zone!!!
1562 * @remarks The guest MSRs should have been saved back into the guest-CPU
1563 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1564 */
1565static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1566{
1567 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1568 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1569
1570 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1571 {
1572 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1573 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1574 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1575 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1576 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1577 }
1578 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1579}
1580#endif /* HC_ARCH_BITS == 64 */
1581
1582
1583/**
1584 * Verifies that our cached values of the VMCS controls are all
1585 * consistent with what's actually present in the VMCS.
1586 *
1587 * @returns VBox status code.
1588 * @param pVCpu Pointer to the VMCPU.
1589 */
1590static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1591{
1592 uint32_t u32Val;
1593 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1594 AssertRCReturn(rc, rc);
1595 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1596 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1597
1598 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1599 AssertRCReturn(rc, rc);
1600 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1601 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1602
1603 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1604 AssertRCReturn(rc, rc);
1605 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1606 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1607
1608 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1609 AssertRCReturn(rc, rc);
1610 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1611 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1612
1613 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1614 {
1615 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1616 AssertRCReturn(rc, rc);
1617 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1618 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1619 }
1620
1621 return VINF_SUCCESS;
1622}
1623
1624
1625#ifdef VBOX_STRICT
1626/**
1627 * Verifies that our cached host EFER value has not changed
1628 * since we cached it.
1629 *
1630 * @param pVCpu Pointer to the VMCPU.
1631 */
1632static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1633{
1634 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1635
1636 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1637 {
1638 uint64_t u64Val;
1639 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, &u64Val);
1640 AssertRC(rc);
1641
1642 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1643 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1644 }
1645}
1646
1647
1648/**
1649 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1650 * VMCS are correct.
1651 *
1652 * @param pVCpu Pointer to the VMCPU.
1653 */
1654static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1655{
1656 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1657
1658 /* Verify MSR counts in the VMCS are what we think it should be. */
1659 uint32_t cMsrs;
1660 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1661 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1662
1663 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1664 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1665
1666 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1667 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1668
1669 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1670 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1671 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1672 {
1673 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1674 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1675 pGuestMsr->u32Msr, cMsrs));
1676
1677 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1678 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1679 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1680
1681 /* Verify that the permissions are as expected in the MSR bitmap. */
1682 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1683 {
1684 VMXMSREXITREAD enmRead;
1685 VMXMSREXITWRITE enmWrite;
1686 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1687 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1688 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1689 {
1690 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1691 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1692 }
1693 else
1694 {
1695 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1696 pGuestMsr->u32Msr, cMsrs));
1697 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1698 pGuestMsr->u32Msr, cMsrs));
1699 }
1700 }
1701 }
1702}
1703#endif /* VBOX_STRICT */
1704
1705
1706/**
1707 * Flushes the TLB using EPT.
1708 *
1709 * @returns VBox status code.
1710 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1711 * enmFlush).
1712 * @param enmFlush Type of flush.
1713 *
1714 * @remarks Caller is responsible for making sure this function is called only
1715 * when NestedPaging is supported and providing @a enmFlush that is
1716 * supported by the CPU.
1717 * @remarks Can be called with interrupts disabled.
1718 */
1719static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1720{
1721 uint64_t au64Descriptor[2];
1722 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1723 au64Descriptor[0] = 0;
1724 else
1725 {
1726 Assert(pVCpu);
1727 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1728 }
1729 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1730
1731 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1732 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1733 rc));
1734 if ( RT_SUCCESS(rc)
1735 && pVCpu)
1736 {
1737 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1738 }
1739}
1740
1741
1742/**
1743 * Flushes the TLB using VPID.
1744 *
1745 * @returns VBox status code.
1746 * @param pVM Pointer to the VM.
1747 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1748 * enmFlush).
1749 * @param enmFlush Type of flush.
1750 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1751 * on @a enmFlush).
1752 *
1753 * @remarks Can be called with interrupts disabled.
1754 */
1755static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1756{
1757 NOREF(pVM);
1758 AssertPtr(pVM);
1759 Assert(pVM->hm.s.vmx.fVpid);
1760
1761 uint64_t au64Descriptor[2];
1762 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1763 {
1764 au64Descriptor[0] = 0;
1765 au64Descriptor[1] = 0;
1766 }
1767 else
1768 {
1769 AssertPtr(pVCpu);
1770 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1771 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1772 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1773 au64Descriptor[1] = GCPtr;
1774 }
1775
1776 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1777 AssertMsg(rc == VINF_SUCCESS,
1778 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1779 if ( RT_SUCCESS(rc)
1780 && pVCpu)
1781 {
1782 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1783 }
1784}
1785
1786
1787/**
1788 * Invalidates a guest page by guest virtual address. Only relevant for
1789 * EPT/VPID, otherwise there is nothing really to invalidate.
1790 *
1791 * @returns VBox status code.
1792 * @param pVM Pointer to the VM.
1793 * @param pVCpu Pointer to the VMCPU.
1794 * @param GCVirt Guest virtual address of the page to invalidate.
1795 */
1796VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1797{
1798 AssertPtr(pVM);
1799 AssertPtr(pVCpu);
1800 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1801
1802 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1803 if (!fFlushPending)
1804 {
1805 /*
1806 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1807 * See @bugref{6043} and @bugref{6177}.
1808 *
1809 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1810 * function maybe called in a loop with individual addresses.
1811 */
1812 if (pVM->hm.s.vmx.fVpid)
1813 {
1814 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1815 {
1816 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1817 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1818 }
1819 else
1820 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1821 }
1822 else if (pVM->hm.s.fNestedPaging)
1823 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1824 }
1825
1826 return VINF_SUCCESS;
1827}
1828
1829
1830/**
1831 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1832 * otherwise there is nothing really to invalidate.
1833 *
1834 * @returns VBox status code.
1835 * @param pVM Pointer to the VM.
1836 * @param pVCpu Pointer to the VMCPU.
1837 * @param GCPhys Guest physical address of the page to invalidate.
1838 */
1839VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1840{
1841 NOREF(pVM); NOREF(GCPhys);
1842 LogFlowFunc(("%RGp\n", GCPhys));
1843
1844 /*
1845 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1846 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1847 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1848 */
1849 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1850 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1851 return VINF_SUCCESS;
1852}
1853
1854
1855/**
1856 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1857 * case where neither EPT nor VPID is supported by the CPU.
1858 *
1859 * @param pVM Pointer to the VM.
1860 * @param pVCpu Pointer to the VMCPU.
1861 * @param pCpu Pointer to the global HM struct.
1862 *
1863 * @remarks Called with interrupts disabled.
1864 */
1865static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1866{
1867 AssertPtr(pVCpu);
1868 AssertPtr(pCpu);
1869 NOREF(pVM);
1870
1871 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1872
1873 /** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1874#if 0
1875 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1876 pVCpu->hm.s.TlbShootdown.cPages = 0;
1877#endif
1878
1879 Assert(pCpu->idCpu != NIL_RTCPUID);
1880 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1881 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1882 pVCpu->hm.s.fForceTLBFlush = false;
1883 return;
1884}
1885
1886
1887/**
1888 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1889 *
1890 * @param pVM Pointer to the VM.
1891 * @param pVCpu Pointer to the VMCPU.
1892 * @param pCpu Pointer to the global HM CPU struct.
1893 * @remarks All references to "ASID" in this function pertains to "VPID" in
1894 * Intel's nomenclature. The reason is, to avoid confusion in compare
1895 * statements since the host-CPU copies are named "ASID".
1896 *
1897 * @remarks Called with interrupts disabled.
1898 */
1899static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1900{
1901#ifdef VBOX_WITH_STATISTICS
1902 bool fTlbFlushed = false;
1903# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1904# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1905 if (!fTlbFlushed) \
1906 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1907 } while (0)
1908#else
1909# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1910# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1911#endif
1912
1913 AssertPtr(pVM);
1914 AssertPtr(pCpu);
1915 AssertPtr(pVCpu);
1916 Assert(pCpu->idCpu != NIL_RTCPUID);
1917
1918 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1919 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1920 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1921
1922 /*
1923 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1924 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1925 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1926 */
1927 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1928 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1929 {
1930 ++pCpu->uCurrentAsid;
1931 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1932 {
1933 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1934 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1935 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1936 }
1937
1938 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1939 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1940 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1941
1942 /*
1943 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1944 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1945 */
1946 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1947 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1948 HMVMX_SET_TAGGED_TLB_FLUSHED();
1949 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1950 }
1951
1952 /* Check for explicit TLB shootdowns. */
1953 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1954 {
1955 /*
1956 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1957 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1958 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1959 * but not guest-physical mappings.
1960 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1961 */
1962 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1963 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1964 HMVMX_SET_TAGGED_TLB_FLUSHED();
1965 }
1966
1967 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1968 * where it is commented out. Support individual entry flushing
1969 * someday. */
1970#if 0
1971 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1972 {
1973 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1974
1975 /*
1976 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1977 * as supported by the CPU.
1978 */
1979 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1980 {
1981 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1982 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1983 }
1984 else
1985 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1986
1987 HMVMX_SET_TAGGED_TLB_FLUSHED();
1988 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1989 pVCpu->hm.s.TlbShootdown.cPages = 0;
1990 }
1991#endif
1992
1993 pVCpu->hm.s.fForceTLBFlush = false;
1994
1995 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1996
1997 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1998 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1999 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2000 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2001 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2002 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2003 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2004 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2005 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2006
2007 /* Update VMCS with the VPID. */
2008 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2009 AssertRC(rc);
2010
2011#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2012}
2013
2014
2015/**
2016 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2017 *
2018 * @returns VBox status code.
2019 * @param pVM Pointer to the VM.
2020 * @param pVCpu Pointer to the VMCPU.
2021 * @param pCpu Pointer to the global HM CPU struct.
2022 *
2023 * @remarks Called with interrupts disabled.
2024 */
2025static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2026{
2027 AssertPtr(pVM);
2028 AssertPtr(pVCpu);
2029 AssertPtr(pCpu);
2030 Assert(pCpu->idCpu != NIL_RTCPUID);
2031 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2032 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2033
2034 /*
2035 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2036 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2037 */
2038 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2039 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2040 {
2041 pVCpu->hm.s.fForceTLBFlush = true;
2042 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2043 }
2044
2045 /* Check for explicit TLB shootdown flushes. */
2046 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2047 {
2048 pVCpu->hm.s.fForceTLBFlush = true;
2049 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2050 }
2051
2052 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2053 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2054
2055 if (pVCpu->hm.s.fForceTLBFlush)
2056 {
2057 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2058 pVCpu->hm.s.fForceTLBFlush = false;
2059 }
2060 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2061 * where it is commented out. Support individual entry flushing
2062 * someday. */
2063#if 0
2064 else
2065 {
2066 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2067 {
2068 /* We cannot flush individual entries without VPID support. Flush using EPT. */
2069 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
2070 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2071 }
2072 else
2073 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2074
2075 pVCpu->hm.s.TlbShootdown.cPages = 0;
2076 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2077 }
2078#endif
2079}
2080
2081
2082/**
2083 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2084 *
2085 * @returns VBox status code.
2086 * @param pVM Pointer to the VM.
2087 * @param pVCpu Pointer to the VMCPU.
2088 * @param pCpu Pointer to the global HM CPU struct.
2089 *
2090 * @remarks Called with interrupts disabled.
2091 */
2092static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2093{
2094 AssertPtr(pVM);
2095 AssertPtr(pVCpu);
2096 AssertPtr(pCpu);
2097 Assert(pCpu->idCpu != NIL_RTCPUID);
2098 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2099 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2100
2101 /*
2102 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2103 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2104 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2105 */
2106 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2107 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2108 {
2109 pVCpu->hm.s.fForceTLBFlush = true;
2110 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2111 }
2112
2113 /* Check for explicit TLB shootdown flushes. */
2114 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2115 {
2116 /*
2117 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2118 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2119 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2120 */
2121 pVCpu->hm.s.fForceTLBFlush = true;
2122 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2123 }
2124
2125 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2126 if (pVCpu->hm.s.fForceTLBFlush)
2127 {
2128 ++pCpu->uCurrentAsid;
2129 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2130 {
2131 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2132 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2133 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2134 }
2135
2136 pVCpu->hm.s.fForceTLBFlush = false;
2137 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2138 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2139 if (pCpu->fFlushAsidBeforeUse)
2140 {
2141 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2142 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2143 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2144 {
2145 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2146 pCpu->fFlushAsidBeforeUse = false;
2147 }
2148 else
2149 {
2150 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2151 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2152 }
2153 }
2154 }
2155 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2156 * where it is commented out. Support individual entry flushing
2157 * someday. */
2158#if 0
2159 else
2160 {
2161 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2162 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2163 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2164 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2165
2166 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2167 {
2168 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2169 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2170 {
2171 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2172 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2173 }
2174 else
2175 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2176
2177 pVCpu->hm.s.TlbShootdown.cPages = 0;
2178 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2179 }
2180 else
2181 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2182 }
2183#endif
2184
2185 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2186 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2187 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2188 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2189 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2190 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2191 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2192
2193 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2194 AssertRC(rc);
2195}
2196
2197
2198/**
2199 * Flushes the guest TLB entry based on CPU capabilities.
2200 *
2201 * @param pVCpu Pointer to the VMCPU.
2202 * @param pCpu Pointer to the global HM CPU struct.
2203 */
2204DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2205{
2206#ifdef HMVMX_ALWAYS_FLUSH_TLB
2207 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2208#endif
2209 PVM pVM = pVCpu->CTX_SUFF(pVM);
2210 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2211 {
2212 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2213 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2214 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2215 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2216 default:
2217 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2218 break;
2219 }
2220
2221 /* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2222 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2223
2224 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2225}
2226
2227
2228/**
2229 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2230 * TLB entries from the host TLB before VM-entry.
2231 *
2232 * @returns VBox status code.
2233 * @param pVM Pointer to the VM.
2234 */
2235static int hmR0VmxSetupTaggedTlb(PVM pVM)
2236{
2237 /*
2238 * Determine optimal flush type for Nested Paging.
2239 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2240 * guest execution (see hmR3InitFinalizeR0()).
2241 */
2242 if (pVM->hm.s.fNestedPaging)
2243 {
2244 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2245 {
2246 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2247 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2248 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2249 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2250 else
2251 {
2252 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2253 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2254 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2255 }
2256
2257 /* Make sure the write-back cacheable memory type for EPT is supported. */
2258 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2259 {
2260 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2261 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2262 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2263 }
2264 }
2265 else
2266 {
2267 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2268 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2269 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2270 }
2271 }
2272
2273 /*
2274 * Determine optimal flush type for VPID.
2275 */
2276 if (pVM->hm.s.vmx.fVpid)
2277 {
2278 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2279 {
2280 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2281 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2282 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2283 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2284 else
2285 {
2286 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2287 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2288 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2289 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2290 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2291 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2292 pVM->hm.s.vmx.fVpid = false;
2293 }
2294 }
2295 else
2296 {
2297 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2298 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2299 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2300 pVM->hm.s.vmx.fVpid = false;
2301 }
2302 }
2303
2304 /*
2305 * Setup the handler for flushing tagged-TLBs.
2306 */
2307 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2308 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2309 else if (pVM->hm.s.fNestedPaging)
2310 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2311 else if (pVM->hm.s.vmx.fVpid)
2312 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2313 else
2314 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2315 return VINF_SUCCESS;
2316}
2317
2318
2319/**
2320 * Sets up pin-based VM-execution controls in the VMCS.
2321 *
2322 * @returns VBox status code.
2323 * @param pVM Pointer to the VM.
2324 * @param pVCpu Pointer to the VMCPU.
2325 */
2326static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2327{
2328 AssertPtr(pVM);
2329 AssertPtr(pVCpu);
2330
2331 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2332 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2333
2334 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2335 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2336
2337 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2338 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2339
2340 /* Enable the VMX preemption timer. */
2341 if (pVM->hm.s.vmx.fUsePreemptTimer)
2342 {
2343 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2344 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2345 }
2346
2347 if ((val & zap) != val)
2348 {
2349 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2350 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2351 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2352 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2353 }
2354
2355 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2356 AssertRCReturn(rc, rc);
2357
2358 pVCpu->hm.s.vmx.u32PinCtls = val;
2359 return rc;
2360}
2361
2362
2363/**
2364 * Sets up processor-based VM-execution controls in the VMCS.
2365 *
2366 * @returns VBox status code.
2367 * @param pVM Pointer to the VM.
2368 * @param pVMCPU Pointer to the VMCPU.
2369 */
2370static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2371{
2372 AssertPtr(pVM);
2373 AssertPtr(pVCpu);
2374
2375 int rc = VERR_INTERNAL_ERROR_5;
2376 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2377 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2378
2379 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2380 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2381 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2382 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2383 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2384 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2385 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2386
2387 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2388 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2389 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2390 {
2391 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2392 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2393 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2394 }
2395
2396 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2397 if (!pVM->hm.s.fNestedPaging)
2398 {
2399 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2400 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2401 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2402 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2403 }
2404
2405 /* Use TPR shadowing if supported by the CPU. */
2406 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2407 {
2408 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2409 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2410 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2411 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2412 AssertRCReturn(rc, rc);
2413
2414 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2415 /* CR8 writes cause a VM-exit based on TPR threshold. */
2416 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2417 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2418 }
2419 else
2420 {
2421 /*
2422 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2423 * Set this control only for 64-bit guests.
2424 */
2425 if (pVM->hm.s.fAllow64BitGuests)
2426 {
2427 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2428 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2429 }
2430 }
2431
2432 /* Use MSR-bitmaps if supported by the CPU. */
2433 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2434 {
2435 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2436
2437 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2438 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2439 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2440 AssertRCReturn(rc, rc);
2441
2442 /*
2443 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2444 * automatically using dedicated fields in the VMCS.
2445 */
2446 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2447 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2448 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2449 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2450 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2451
2452#if HC_ARCH_BITS == 64
2453 /*
2454 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2455 */
2456 if (pVM->hm.s.fAllow64BitGuests)
2457 {
2458 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2459 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2460 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2461 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2462 }
2463#endif
2464 }
2465
2466 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2467 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2468 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2469
2470 if ((val & zap) != val)
2471 {
2472 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2473 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2474 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2475 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2476 }
2477
2478 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2479 AssertRCReturn(rc, rc);
2480
2481 pVCpu->hm.s.vmx.u32ProcCtls = val;
2482
2483 /*
2484 * Secondary processor-based VM-execution controls.
2485 */
2486 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2487 {
2488 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2489 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2490
2491 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2492 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2493
2494 if (pVM->hm.s.fNestedPaging)
2495 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2496 else
2497 {
2498 /*
2499 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2500 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2501 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2502 */
2503 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2504 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2505 }
2506
2507 if (pVM->hm.s.vmx.fVpid)
2508 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2509
2510 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2511 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2512
2513 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2514 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2515 * done dynamically. */
2516 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2517 {
2518 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2519 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2520 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2521 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2522 AssertRCReturn(rc, rc);
2523 }
2524
2525 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2526 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2527
2528 if ((val & zap) != val)
2529 {
2530 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2531 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2532 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2533 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2534 }
2535
2536 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2537 AssertRCReturn(rc, rc);
2538
2539 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2540 }
2541 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2542 {
2543 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2544 "available\n"));
2545 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2546 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2547 }
2548
2549 return VINF_SUCCESS;
2550}
2551
2552
2553/**
2554 * Sets up miscellaneous (everything other than Pin & Processor-based
2555 * VM-execution) control fields in the VMCS.
2556 *
2557 * @returns VBox status code.
2558 * @param pVM Pointer to the VM.
2559 * @param pVCpu Pointer to the VMCPU.
2560 */
2561static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2562{
2563 NOREF(pVM);
2564 AssertPtr(pVM);
2565 AssertPtr(pVCpu);
2566
2567 int rc = VERR_GENERAL_FAILURE;
2568
2569 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2570#if 0
2571 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2572 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2573 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2574
2575 /*
2576 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2577 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2578 * We thus use the exception bitmap to control it rather than use both.
2579 */
2580 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2581 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2582
2583 /** @todo Explore possibility of using IO-bitmaps. */
2584 /* All IO & IOIO instructions cause VM-exits. */
2585 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2586 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2587
2588 /* Initialize the MSR-bitmap area. */
2589 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2590 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2591 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2592#endif
2593
2594 /* Setup MSR auto-load/store area. */
2595 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2596 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2597 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2598 AssertRCReturn(rc, rc);
2599 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2600 AssertRCReturn(rc, rc);
2601
2602 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2603 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2604 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2605 AssertRCReturn(rc, rc);
2606
2607 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2608 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2609 AssertRCReturn(rc, rc);
2610
2611 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2612#if 0
2613 /* Setup debug controls */
2614 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2615 AssertRCReturn(rc, rc);
2616 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2617 AssertRCReturn(rc, rc);
2618#endif
2619
2620 return rc;
2621}
2622
2623
2624/**
2625 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2626 *
2627 * @returns VBox status code.
2628 * @param pVM Pointer to the VM.
2629 * @param pVCpu Pointer to the VMCPU.
2630 */
2631static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2632{
2633 AssertPtr(pVM);
2634 AssertPtr(pVCpu);
2635
2636 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2637
2638 uint32_t u32XcptBitmap = 0;
2639
2640 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2641 if (!pVM->hm.s.fNestedPaging)
2642 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2643
2644 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2645 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2646 AssertRCReturn(rc, rc);
2647 return rc;
2648}
2649
2650
2651/**
2652 * Sets up the initial guest-state mask. The guest-state mask is consulted
2653 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2654 * for the nested virtualization case (as it would cause a VM-exit).
2655 *
2656 * @param pVCpu Pointer to the VMCPU.
2657 */
2658static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2659{
2660 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2661 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2662 return VINF_SUCCESS;
2663}
2664
2665
2666/**
2667 * Does per-VM VT-x initialization.
2668 *
2669 * @returns VBox status code.
2670 * @param pVM Pointer to the VM.
2671 */
2672VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2673{
2674 LogFlowFunc(("pVM=%p\n", pVM));
2675
2676 int rc = hmR0VmxStructsAlloc(pVM);
2677 if (RT_FAILURE(rc))
2678 {
2679 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2680 return rc;
2681 }
2682
2683 return VINF_SUCCESS;
2684}
2685
2686
2687/**
2688 * Does per-VM VT-x termination.
2689 *
2690 * @returns VBox status code.
2691 * @param pVM Pointer to the VM.
2692 */
2693VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2694{
2695 LogFlowFunc(("pVM=%p\n", pVM));
2696
2697#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2698 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2699 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2700#endif
2701 hmR0VmxStructsFree(pVM);
2702 return VINF_SUCCESS;
2703}
2704
2705
2706/**
2707 * Sets up the VM for execution under VT-x.
2708 * This function is only called once per-VM during initialization.
2709 *
2710 * @returns VBox status code.
2711 * @param pVM Pointer to the VM.
2712 */
2713VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2714{
2715 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2716 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2717
2718 LogFlowFunc(("pVM=%p\n", pVM));
2719
2720 /*
2721 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2722 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2723 */
2724 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2725 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2726 || !pVM->hm.s.vmx.pRealModeTSS))
2727 {
2728 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2729 return VERR_INTERNAL_ERROR;
2730 }
2731
2732#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2733 /*
2734 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2735 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2736 */
2737 if ( pVM->hm.s.fAllow64BitGuests
2738 && !HMVMX_IS_64BIT_HOST_MODE())
2739 {
2740 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2741 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2742 }
2743#endif
2744
2745 /* Initialize these always, see hmR3InitFinalizeR0().*/
2746 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2747 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2748
2749 /* Setup the tagged-TLB flush handlers. */
2750 int rc = hmR0VmxSetupTaggedTlb(pVM);
2751 if (RT_FAILURE(rc))
2752 {
2753 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2754 return rc;
2755 }
2756
2757 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2758 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2759#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2760 if ( HMVMX_IS_64BIT_HOST_MODE()
2761 && (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2762 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2763 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2764 {
2765 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2766 }
2767#endif
2768
2769 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2770 {
2771 PVMCPU pVCpu = &pVM->aCpus[i];
2772 AssertPtr(pVCpu);
2773 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2774
2775 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2776 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2777
2778 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2779 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2780 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2781
2782 /* Set revision dword at the beginning of the VMCS structure. */
2783 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2784
2785 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2786 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2787 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2788 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2789
2790 /* Load this VMCS as the current VMCS. */
2791 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2792 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2793 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2794
2795 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2796 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2797 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2798
2799 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2800 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2801 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2802
2803 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2804 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2805 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2806
2807 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2808 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2809 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2810
2811 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2812 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2813 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2814
2815#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2816 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2817 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2818 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2819#endif
2820
2821 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2822 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2823 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2824 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2825
2826 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2827
2828 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2829 }
2830
2831 return VINF_SUCCESS;
2832}
2833
2834
2835/**
2836 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2837 * the VMCS.
2838 *
2839 * @returns VBox status code.
2840 * @param pVM Pointer to the VM.
2841 * @param pVCpu Pointer to the VMCPU.
2842 */
2843DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2844{
2845 NOREF(pVM); NOREF(pVCpu);
2846
2847 RTCCUINTREG uReg = ASMGetCR0();
2848 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2849 AssertRCReturn(rc, rc);
2850
2851#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2852 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2853 if (HMVMX_IS_64BIT_HOST_MODE())
2854 {
2855 uint64_t uRegCR3 = HMR0Get64bitCR3();
2856 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2857 }
2858 else
2859#endif
2860 {
2861 uReg = ASMGetCR3();
2862 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2863 }
2864 AssertRCReturn(rc, rc);
2865
2866 uReg = ASMGetCR4();
2867 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2868 AssertRCReturn(rc, rc);
2869 return rc;
2870}
2871
2872
2873#if HC_ARCH_BITS == 64
2874/**
2875 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2876 * requirements. See hmR0VmxSaveHostSegmentRegs().
2877 */
2878# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2879 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2880 { \
2881 bool fValidSelector = true; \
2882 if ((selValue) & X86_SEL_LDT) \
2883 { \
2884 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2885 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2886 } \
2887 if (fValidSelector) \
2888 { \
2889 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2890 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2891 } \
2892 (selValue) = 0; \
2893 }
2894#endif
2895
2896
2897/**
2898 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2899 * the host-state area in the VMCS.
2900 *
2901 * @returns VBox status code.
2902 * @param pVM Pointer to the VM.
2903 * @param pVCpu Pointer to the VMCPU.
2904 */
2905DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2906{
2907 int rc = VERR_INTERNAL_ERROR_5;
2908
2909#if HC_ARCH_BITS == 64
2910 /*
2911 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2912 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2913 */
2914 AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2915 ("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2916#endif
2917
2918 /*
2919 * Host DS, ES, FS and GS segment registers.
2920 */
2921#if HC_ARCH_BITS == 64
2922 RTSEL uSelDS = ASMGetDS();
2923 RTSEL uSelES = ASMGetES();
2924 RTSEL uSelFS = ASMGetFS();
2925 RTSEL uSelGS = ASMGetGS();
2926#else
2927 RTSEL uSelDS = 0;
2928 RTSEL uSelES = 0;
2929 RTSEL uSelFS = 0;
2930 RTSEL uSelGS = 0;
2931#endif
2932
2933 /* Recalculate which host-state bits need to be manually restored. */
2934 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2935
2936 /*
2937 * Host CS and SS segment registers.
2938 */
2939#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2940 RTSEL uSelCS;
2941 RTSEL uSelSS;
2942 if (HMVMX_IS_64BIT_HOST_MODE())
2943 {
2944 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2945 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2946 }
2947 else
2948 {
2949 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2950 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2951 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2952 }
2953#else
2954 RTSEL uSelCS = ASMGetCS();
2955 RTSEL uSelSS = ASMGetSS();
2956#endif
2957
2958 /*
2959 * Host TR segment register.
2960 */
2961 RTSEL uSelTR = ASMGetTR();
2962
2963#if HC_ARCH_BITS == 64
2964 /*
2965 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2966 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2967 */
2968 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2969 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2970 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2971 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2972# undef VMXLOCAL_ADJUST_HOST_SEG
2973#endif
2974
2975 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2976 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2977 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2978 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2979 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2980 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2981 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2982 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2983 Assert(uSelCS);
2984 Assert(uSelTR);
2985
2986 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2987#if 0
2988 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2989 Assert(uSelSS != 0);
2990#endif
2991
2992 /* Write these host selector fields into the host-state area in the VMCS. */
2993 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2994 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2995#if HC_ARCH_BITS == 64
2996 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2997 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2998 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2999 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
3000#endif
3001 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
3002
3003 /*
3004 * Host GDTR and IDTR.
3005 */
3006 RTGDTR Gdtr;
3007 RT_ZERO(Gdtr);
3008#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3009 if (HMVMX_IS_64BIT_HOST_MODE())
3010 {
3011 X86XDTR64 Gdtr64;
3012 X86XDTR64 Idtr64;
3013 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
3014 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
3015 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
3016
3017 Gdtr.cbGdt = Gdtr64.cb;
3018 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
3019 }
3020 else
3021#endif
3022 {
3023 RTIDTR Idtr;
3024 ASMGetGDTR(&Gdtr);
3025 ASMGetIDTR(&Idtr);
3026 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
3027 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
3028
3029#if HC_ARCH_BITS == 64
3030 /*
3031 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3032 * maximum limit (0xffff) on every VM-exit.
3033 */
3034 if (Gdtr.cbGdt != 0xffff)
3035 {
3036 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3037 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3038 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3039 }
3040
3041 /*
3042 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3043 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3044 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3045 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3046 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3047 * hosts where we are pretty sure it won't cause trouble.
3048 */
3049# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3050 if (Idtr.cbIdt < 0x0fff)
3051# else
3052 if (Idtr.cbIdt != 0xffff)
3053# endif
3054 {
3055 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3056 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3057 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3058 }
3059#endif
3060 }
3061
3062 /*
3063 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3064 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3065 */
3066 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3067 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3068 VERR_VMX_INVALID_HOST_STATE);
3069
3070 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3071#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3072 if (HMVMX_IS_64BIT_HOST_MODE())
3073 {
3074 /* We need the 64-bit TR base for hybrid darwin. */
3075 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
3076 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
3077 }
3078 else
3079#endif
3080 {
3081 uintptr_t uTRBase;
3082#if HC_ARCH_BITS == 64
3083 uTRBase = X86DESC64_BASE(pDesc);
3084
3085 /*
3086 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3087 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3088 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3089 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3090 *
3091 * [1] See Intel spec. 3.5 "System Descriptor Types".
3092 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3093 */
3094 Assert(pDesc->System.u4Type == 11);
3095 if ( pDesc->System.u16LimitLow != 0x67
3096 || pDesc->System.u4LimitHigh)
3097 {
3098 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3099 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3100 if (pVM->hm.s.uHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3101 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3102 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3103
3104 /* Store the GDTR here as we need it while restoring TR. */
3105 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3106 }
3107#else
3108 uTRBase = X86DESC_BASE(pDesc);
3109#endif
3110 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3111 }
3112 AssertRCReturn(rc, rc);
3113
3114 /*
3115 * Host FS base and GS base.
3116 */
3117#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3118 if (HMVMX_IS_64BIT_HOST_MODE())
3119 {
3120 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3121 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3122 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
3123 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
3124
3125# if HC_ARCH_BITS == 64
3126 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3127 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3128 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3129 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3130 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3131# endif
3132 }
3133#endif
3134 return rc;
3135}
3136
3137
3138/**
3139 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3140 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3141 * the host after every successful VM-exit.
3142 *
3143 * @returns VBox status code.
3144 * @param pVM Pointer to the VM.
3145 * @param pVCpu Pointer to the VMCPU.
3146 *
3147 * @remarks No-long-jump zone!!!
3148 */
3149DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3150{
3151 NOREF(pVM);
3152
3153 AssertPtr(pVCpu);
3154 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3155
3156 int rc = VINF_SUCCESS;
3157#if HC_ARCH_BITS == 64
3158 if (pVM->hm.s.fAllow64BitGuests)
3159 hmR0VmxLazySaveHostMsrs(pVCpu);
3160#endif
3161
3162 /*
3163 * Host Sysenter MSRs.
3164 */
3165 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3166 AssertRCReturn(rc, rc);
3167#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3168 if (HMVMX_IS_64BIT_HOST_MODE())
3169 {
3170 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3171 AssertRCReturn(rc, rc);
3172 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3173 }
3174 else
3175 {
3176 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3177 AssertRCReturn(rc, rc);
3178 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3179 }
3180#elif HC_ARCH_BITS == 32
3181 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3182 AssertRCReturn(rc, rc);
3183 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3184#else
3185 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3186 AssertRCReturn(rc, rc);
3187 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3188#endif
3189 AssertRCReturn(rc, rc);
3190
3191 /*
3192 * Host EFER MSR.
3193 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3194 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3195 */
3196 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3197 {
3198 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3199 AssertRCReturn(rc, rc);
3200 }
3201
3202 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3203 * hmR0VmxLoadGuestExitCtls() !! */
3204
3205 return rc;
3206}
3207
3208
3209/**
3210 * Figures out if we need to swap the EFER MSR which is
3211 * particularly expensive.
3212 *
3213 * We check all relevant bits. For now, that's everything
3214 * besides LMA/LME, as these two bits are handled by VM-entry,
3215 * see hmR0VmxLoadGuestExitCtls() and
3216 * hmR0VMxLoadGuestEntryCtls().
3217 *
3218 * @returns true if we need to load guest EFER, false otherwise.
3219 * @param pVCpu Pointer to the VMCPU.
3220 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3221 * out-of-sync. Make sure to update the required fields
3222 * before using them.
3223 *
3224 * @remarks Requires EFER, CR4.
3225 * @remarks No-long-jump zone!!!
3226 */
3227static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3228{
3229#ifdef HMVMX_ALWAYS_SWAP_EFER
3230 return true;
3231#endif
3232
3233#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3234 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3235 if (CPUMIsGuestInLongMode(pVCpu))
3236 return false;
3237#endif
3238
3239 PVM pVM = pVCpu->CTX_SUFF(pVM);
3240 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3241 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3242
3243 /*
3244 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3245 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3246 */
3247 if ( CPUMIsGuestInLongMode(pVCpu)
3248 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3249 {
3250 return true;
3251 }
3252
3253 /*
3254 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3255 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3256 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3257 */
3258 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3259 && (pMixedCtx->cr0 & X86_CR0_PG)
3260 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3261 {
3262 /* Assert that host is PAE capable. */
3263 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3264 return true;
3265 }
3266
3267 /** @todo Check the latest Intel spec. for any other bits,
3268 * like SMEP/SMAP? */
3269 return false;
3270}
3271
3272
3273/**
3274 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3275 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3276 * controls".
3277 *
3278 * @returns VBox status code.
3279 * @param pVCpu Pointer to the VMCPU.
3280 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3281 * out-of-sync. Make sure to update the required fields
3282 * before using them.
3283 *
3284 * @remarks Requires EFER.
3285 * @remarks No-long-jump zone!!!
3286 */
3287DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3288{
3289 int rc = VINF_SUCCESS;
3290 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3291 {
3292 PVM pVM = pVCpu->CTX_SUFF(pVM);
3293 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3294 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3295
3296 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3297 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3298
3299 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3300 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3301 {
3302 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3303 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3304 }
3305 else
3306 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3307
3308 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3309 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3310 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3311 {
3312 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3313 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3314 }
3315
3316 /*
3317 * The following should -not- be set (since we're not in SMM mode):
3318 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3319 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3320 */
3321
3322 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3323 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3324
3325 if ((val & zap) != val)
3326 {
3327 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3328 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3329 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3330 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3331 }
3332
3333 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3334 AssertRCReturn(rc, rc);
3335
3336 pVCpu->hm.s.vmx.u32EntryCtls = val;
3337 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3338 }
3339 return rc;
3340}
3341
3342
3343/**
3344 * Sets up the VM-exit controls in the VMCS.
3345 *
3346 * @returns VBox status code.
3347 * @param pVM Pointer to the VM.
3348 * @param pVCpu Pointer to the VMCPU.
3349 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3350 * out-of-sync. Make sure to update the required fields
3351 * before using them.
3352 *
3353 * @remarks Requires EFER.
3354 */
3355DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3356{
3357 NOREF(pMixedCtx);
3358
3359 int rc = VINF_SUCCESS;
3360 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3361 {
3362 PVM pVM = pVCpu->CTX_SUFF(pVM);
3363 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3364 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3365
3366 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3367 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3368
3369 /*
3370 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3371 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3372 */
3373#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3374 if (HMVMX_IS_64BIT_HOST_MODE())
3375 {
3376 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3377 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3378 }
3379 else
3380 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3381#else
3382 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3383 {
3384 /* The switcher returns to long mode, EFER is managed by the switcher. */
3385 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3386 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3387 }
3388 else
3389 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3390#endif /* HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
3391
3392 /* If the newer VMCS fields for managing EFER exists, use it. */
3393 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3394 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3395 {
3396 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3397 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3398 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3399 }
3400
3401 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3402 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3403
3404 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3405 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3406 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3407
3408 if ( pVM->hm.s.vmx.fUsePreemptTimer
3409 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3410 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3411
3412 if ((val & zap) != val)
3413 {
3414 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3415 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3416 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3417 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3418 }
3419
3420 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3421 AssertRCReturn(rc, rc);
3422
3423 pVCpu->hm.s.vmx.u32ExitCtls = val;
3424 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3425 }
3426 return rc;
3427}
3428
3429
3430/**
3431 * Loads the guest APIC and related state.
3432 *
3433 * @returns VBox status code.
3434 * @param pVM Pointer to the VM.
3435 * @param pVCpu Pointer to the VMCPU.
3436 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3437 * out-of-sync. Make sure to update the required fields
3438 * before using them.
3439 */
3440DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3441{
3442 NOREF(pMixedCtx);
3443
3444 int rc = VINF_SUCCESS;
3445 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3446 {
3447 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3448 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3449 {
3450 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3451
3452 bool fPendingIntr = false;
3453 uint8_t u8Tpr = 0;
3454 uint8_t u8PendingIntr = 0;
3455 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3456 AssertRCReturn(rc, rc);
3457
3458 /*
3459 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3460 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3461 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3462 * the interrupt when we VM-exit for other reasons.
3463 */
3464 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3465 uint32_t u32TprThreshold = 0;
3466 if (fPendingIntr)
3467 {
3468 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3469 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3470 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3471 if (u8PendingPriority <= u8TprPriority)
3472 u32TprThreshold = u8PendingPriority;
3473 else
3474 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3475 }
3476 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3477
3478 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3479 AssertRCReturn(rc, rc);
3480 }
3481
3482 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3483 }
3484 return rc;
3485}
3486
3487
3488/**
3489 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3490 *
3491 * @returns Guest's interruptibility-state.
3492 * @param pVCpu Pointer to the VMCPU.
3493 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3494 * out-of-sync. Make sure to update the required fields
3495 * before using them.
3496 *
3497 * @remarks No-long-jump zone!!!
3498 */
3499DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3500{
3501 /*
3502 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3503 */
3504 uint32_t uIntrState = 0;
3505 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3506 {
3507 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3508 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3509 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3510 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3511 {
3512 if (pMixedCtx->eflags.Bits.u1IF)
3513 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3514 else
3515 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3516 }
3517 /* else: Although we can clear the force-flag here, let's keep this side-effects free. */
3518 }
3519
3520 /*
3521 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3522 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3523 * setting this would block host-NMIs and IRET will not clear the blocking.
3524 *
3525 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3526 */
3527 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3528 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3529 {
3530 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3531 }
3532
3533 return uIntrState;
3534}
3535
3536
3537/**
3538 * Loads the guest's interruptibility-state into the guest-state area in the
3539 * VMCS.
3540 *
3541 * @returns VBox status code.
3542 * @param pVCpu Pointer to the VMCPU.
3543 * @param uIntrState The interruptibility-state to set.
3544 */
3545static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3546{
3547 NOREF(pVCpu);
3548 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3549 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3550 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3551 AssertRCReturn(rc, rc);
3552 return rc;
3553}
3554
3555
3556/**
3557 * Loads the guest's RIP into the guest-state area in the VMCS.
3558 *
3559 * @returns VBox status code.
3560 * @param pVCpu Pointer to the VMCPU.
3561 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3562 * out-of-sync. Make sure to update the required fields
3563 * before using them.
3564 *
3565 * @remarks No-long-jump zone!!!
3566 */
3567static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3568{
3569 int rc = VINF_SUCCESS;
3570 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3571 {
3572 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3573 AssertRCReturn(rc, rc);
3574
3575 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3576 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3577 HMCPU_CF_VALUE(pVCpu)));
3578 }
3579 return rc;
3580}
3581
3582
3583/**
3584 * Loads the guest's RSP into the guest-state area in the VMCS.
3585 *
3586 * @returns VBox status code.
3587 * @param pVCpu Pointer to the VMCPU.
3588 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3589 * out-of-sync. Make sure to update the required fields
3590 * before using them.
3591 *
3592 * @remarks No-long-jump zone!!!
3593 */
3594static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3595{
3596 int rc = VINF_SUCCESS;
3597 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3598 {
3599 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3600 AssertRCReturn(rc, rc);
3601
3602 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3603 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3604 }
3605 return rc;
3606}
3607
3608
3609/**
3610 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3611 *
3612 * @returns VBox status code.
3613 * @param pVCpu Pointer to the VMCPU.
3614 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3615 * out-of-sync. Make sure to update the required fields
3616 * before using them.
3617 *
3618 * @remarks No-long-jump zone!!!
3619 */
3620static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3621{
3622 int rc = VINF_SUCCESS;
3623 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3624 {
3625 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3626 Let us assert it as such and use 32-bit VMWRITE. */
3627 Assert(!(pMixedCtx->rflags.u64 >> 32));
3628 X86EFLAGS Eflags = pMixedCtx->eflags;
3629 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3630 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3631 * These will never be cleared/set, unless some other part of the VMM
3632 * code is buggy - in which case we're better of finding and fixing
3633 * those bugs than hiding them. */
3634 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3635 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3636 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3637 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3638
3639 /*
3640 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3641 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3642 */
3643 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3644 {
3645 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3646 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3647 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3648 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3649 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3650 }
3651
3652 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3653 AssertRCReturn(rc, rc);
3654
3655 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3656 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3657 }
3658 return rc;
3659}
3660
3661
3662/**
3663 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3664 *
3665 * @returns VBox status code.
3666 * @param pVCpu Pointer to the VMCPU.
3667 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3668 * out-of-sync. Make sure to update the required fields
3669 * before using them.
3670 *
3671 * @remarks No-long-jump zone!!!
3672 */
3673DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3674{
3675 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3676 AssertRCReturn(rc, rc);
3677 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3678 AssertRCReturn(rc, rc);
3679 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3680 AssertRCReturn(rc, rc);
3681 return rc;
3682}
3683
3684
3685/**
3686 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3687 * CR0 is partially shared with the host and we have to consider the FPU bits.
3688 *
3689 * @returns VBox status code.
3690 * @param pVM Pointer to the VM.
3691 * @param pVCpu Pointer to the VMCPU.
3692 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3693 * out-of-sync. Make sure to update the required fields
3694 * before using them.
3695 *
3696 * @remarks No-long-jump zone!!!
3697 */
3698static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3699{
3700 /*
3701 * Guest CR0.
3702 * Guest FPU.
3703 */
3704 int rc = VINF_SUCCESS;
3705 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3706 {
3707 Assert(!(pMixedCtx->cr0 >> 32));
3708 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3709 PVM pVM = pVCpu->CTX_SUFF(pVM);
3710
3711 /* The guest's view (read access) of its CR0 is unblemished. */
3712 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3713 AssertRCReturn(rc, rc);
3714 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3715
3716 /* Setup VT-x's view of the guest CR0. */
3717 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3718 if (pVM->hm.s.fNestedPaging)
3719 {
3720 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3721 {
3722 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3723 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3724 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3725 }
3726 else
3727 {
3728 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3729 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3730 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3731 }
3732
3733 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3734 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3735 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3736
3737 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3738 AssertRCReturn(rc, rc);
3739 }
3740 else
3741 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3742
3743 /*
3744 * Guest FPU bits.
3745 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3746 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3747 */
3748 u32GuestCR0 |= X86_CR0_NE;
3749 bool fInterceptNM = false;
3750 if (CPUMIsGuestFPUStateActive(pVCpu))
3751 {
3752 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3753 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3754 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3755 }
3756 else
3757 {
3758 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3759 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3760 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3761 }
3762
3763 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3764 bool fInterceptMF = false;
3765 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3766 fInterceptMF = true;
3767
3768 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3769 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3770 {
3771 Assert(PDMVmmDevHeapIsEnabled(pVM));
3772 Assert(pVM->hm.s.vmx.pRealModeTSS);
3773 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3774 fInterceptNM = true;
3775 fInterceptMF = true;
3776 }
3777 else
3778 {
3779 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626} comment #11. */
3780 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3781 }
3782
3783 if (fInterceptNM)
3784 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3785 else
3786 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3787
3788 if (fInterceptMF)
3789 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3790 else
3791 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3792
3793 /* Additional intercepts for debugging, define these yourself explicitly. */
3794#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3795 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3796 | RT_BIT(X86_XCPT_BP)
3797 | RT_BIT(X86_XCPT_DB)
3798 | RT_BIT(X86_XCPT_DE)
3799 | RT_BIT(X86_XCPT_NM)
3800 | RT_BIT(X86_XCPT_TS)
3801 | RT_BIT(X86_XCPT_UD)
3802 | RT_BIT(X86_XCPT_NP)
3803 | RT_BIT(X86_XCPT_SS)
3804 | RT_BIT(X86_XCPT_GP)
3805 | RT_BIT(X86_XCPT_PF)
3806 | RT_BIT(X86_XCPT_MF)
3807 ;
3808#elif defined(HMVMX_ALWAYS_TRAP_PF)
3809 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3810#endif
3811
3812 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3813
3814 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3815 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3816 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3817 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3818 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3819 else
3820 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3821
3822 u32GuestCR0 |= uSetCR0;
3823 u32GuestCR0 &= uZapCR0;
3824 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3825
3826 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3827 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3828 AssertRCReturn(rc, rc);
3829 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3830 AssertRCReturn(rc, rc);
3831 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3832 uZapCR0));
3833
3834 /*
3835 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3836 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3837 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3838 */
3839 uint32_t u32CR0Mask = 0;
3840 u32CR0Mask = X86_CR0_PE
3841 | X86_CR0_NE
3842 | X86_CR0_WP
3843 | X86_CR0_PG
3844 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3845 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3846 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3847
3848 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3849 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3850 * and @bugref{6944}. */
3851#if 0
3852 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3853 u32CR0Mask &= ~X86_CR0_PE;
3854#endif
3855 if (pVM->hm.s.fNestedPaging)
3856 u32CR0Mask &= ~X86_CR0_WP;
3857
3858 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3859 if (fInterceptNM)
3860 {
3861 u32CR0Mask |= X86_CR0_TS
3862 | X86_CR0_MP;
3863 }
3864
3865 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3866 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3867 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3868 AssertRCReturn(rc, rc);
3869 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3870
3871 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3872 }
3873 return rc;
3874}
3875
3876
3877/**
3878 * Loads the guest control registers (CR3, CR4) into the guest-state area
3879 * in the VMCS.
3880 *
3881 * @returns VBox status code.
3882 * @param pVM Pointer to the VM.
3883 * @param pVCpu Pointer to the VMCPU.
3884 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3885 * out-of-sync. Make sure to update the required fields
3886 * before using them.
3887 *
3888 * @remarks No-long-jump zone!!!
3889 */
3890static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3891{
3892 int rc = VINF_SUCCESS;
3893 PVM pVM = pVCpu->CTX_SUFF(pVM);
3894
3895 /*
3896 * Guest CR2.
3897 * It's always loaded in the assembler code. Nothing to do here.
3898 */
3899
3900 /*
3901 * Guest CR3.
3902 */
3903 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3904 {
3905 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3906 if (pVM->hm.s.fNestedPaging)
3907 {
3908 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3909
3910 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3911 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3912 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3913 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3914
3915 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3916 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3917 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3918
3919 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3920 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3921 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3922 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3923
3924 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3925 AssertRCReturn(rc, rc);
3926 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3927
3928 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3929 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3930 {
3931 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3932 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3933 {
3934 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3935 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3936 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3937 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3938 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3939 }
3940
3941 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3942 have Unrestricted Execution to handle the guest when it's not using paging. */
3943 GCPhysGuestCR3 = pMixedCtx->cr3;
3944 }
3945 else
3946 {
3947 /*
3948 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3949 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3950 * EPT takes care of translating it to host-physical addresses.
3951 */
3952 RTGCPHYS GCPhys;
3953 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3954 Assert(PDMVmmDevHeapIsEnabled(pVM));
3955
3956 /* We obtain it here every time as the guest could have relocated this PCI region. */
3957 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3958 AssertRCReturn(rc, rc);
3959
3960 GCPhysGuestCR3 = GCPhys;
3961 }
3962
3963 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
3964 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3965 }
3966 else
3967 {
3968 /* Non-nested paging case, just use the hypervisor's CR3. */
3969 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3970
3971 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
3972 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3973 }
3974 AssertRCReturn(rc, rc);
3975
3976 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3977 }
3978
3979 /*
3980 * Guest CR4.
3981 */
3982 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3983 {
3984 Assert(!(pMixedCtx->cr4 >> 32));
3985 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3986
3987 /* The guest's view of its CR4 is unblemished. */
3988 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3989 AssertRCReturn(rc, rc);
3990 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
3991
3992 /* Setup VT-x's view of the guest CR4. */
3993 /*
3994 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3995 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3996 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3997 */
3998 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3999 {
4000 Assert(pVM->hm.s.vmx.pRealModeTSS);
4001 Assert(PDMVmmDevHeapIsEnabled(pVM));
4002 u32GuestCR4 &= ~X86_CR4_VME;
4003 }
4004
4005 if (pVM->hm.s.fNestedPaging)
4006 {
4007 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4008 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4009 {
4010 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4011 u32GuestCR4 |= X86_CR4_PSE;
4012 /* Our identity mapping is a 32-bit page directory. */
4013 u32GuestCR4 &= ~X86_CR4_PAE;
4014 }
4015 /* else use guest CR4.*/
4016 }
4017 else
4018 {
4019 /*
4020 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4021 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4022 */
4023 switch (pVCpu->hm.s.enmShadowMode)
4024 {
4025 case PGMMODE_REAL: /* Real-mode. */
4026 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4027 case PGMMODE_32_BIT: /* 32-bit paging. */
4028 {
4029 u32GuestCR4 &= ~X86_CR4_PAE;
4030 break;
4031 }
4032
4033 case PGMMODE_PAE: /* PAE paging. */
4034 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4035 {
4036 u32GuestCR4 |= X86_CR4_PAE;
4037 break;
4038 }
4039
4040 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4041 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4042#ifdef VBOX_ENABLE_64_BITS_GUESTS
4043 break;
4044#endif
4045 default:
4046 AssertFailed();
4047 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4048 }
4049 }
4050
4051 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4052 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4053 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4054 u32GuestCR4 |= uSetCR4;
4055 u32GuestCR4 &= uZapCR4;
4056
4057 /* Write VT-x's view of the guest CR4 into the VMCS. */
4058 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4059 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4060 AssertRCReturn(rc, rc);
4061
4062 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4063 uint32_t u32CR4Mask = 0;
4064 u32CR4Mask = X86_CR4_VME
4065 | X86_CR4_PAE
4066 | X86_CR4_PGE
4067 | X86_CR4_PSE
4068 | X86_CR4_VMXE;
4069 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4070 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4071 AssertRCReturn(rc, rc);
4072
4073 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4074 }
4075 return rc;
4076}
4077
4078
4079/**
4080 * Loads the guest debug registers into the guest-state area in the VMCS.
4081 * This also sets up whether #DB and MOV DRx accesses cause VM-exits.
4082 *
4083 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4084 *
4085 * @returns VBox status code.
4086 * @param pVCpu Pointer to the VMCPU.
4087 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4088 * out-of-sync. Make sure to update the required fields
4089 * before using them.
4090 *
4091 * @remarks No-long-jump zone!!!
4092 */
4093static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4094{
4095 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4096 return VINF_SUCCESS;
4097
4098#ifdef VBOX_STRICT
4099 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4100 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4101 {
4102 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4103 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4104 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4105 }
4106#endif
4107
4108 int rc;
4109 PVM pVM = pVCpu->CTX_SUFF(pVM);
4110 bool fInterceptDB = false;
4111 bool fInterceptMovDRx = false;
4112 if ( pVCpu->hm.s.fSingleInstruction
4113 || DBGFIsStepping(pVCpu))
4114 {
4115 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4116 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4117 {
4118 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4119 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4120 AssertRCReturn(rc, rc);
4121 Assert(fInterceptDB == false);
4122 }
4123 else
4124 {
4125 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4126 pVCpu->hm.s.fClearTrapFlag = true;
4127 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4128 fInterceptDB = true;
4129 }
4130 }
4131
4132 if ( fInterceptDB
4133 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4134 {
4135 /*
4136 * Use the combined guest and host DRx values found in the hypervisor
4137 * register set because the debugger has breakpoints active or someone
4138 * is single stepping on the host side without a monitor trap flag.
4139 *
4140 * Note! DBGF expects a clean DR6 state before executing guest code.
4141 */
4142#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4143 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4144 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4145 {
4146 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4147 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4148 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4149 }
4150 else
4151#endif
4152 if (!CPUMIsHyperDebugStateActive(pVCpu))
4153 {
4154 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4155 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4156 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4157 }
4158
4159 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4160 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4161 AssertRCReturn(rc, rc);
4162
4163 pVCpu->hm.s.fUsingHyperDR7 = true;
4164 fInterceptDB = true;
4165 fInterceptMovDRx = true;
4166 }
4167 else
4168 {
4169 /*
4170 * If the guest has enabled debug registers, we need to load them prior to
4171 * executing guest code so they'll trigger at the right time.
4172 */
4173 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4174 {
4175#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4176 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4177 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4178 {
4179 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4180 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4181 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4182 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4183 }
4184 else
4185#endif
4186 if (!CPUMIsGuestDebugStateActive(pVCpu))
4187 {
4188 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4189 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4190 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4191 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4192 }
4193 Assert(!fInterceptDB);
4194 Assert(!fInterceptMovDRx);
4195 }
4196 /*
4197 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4198 * must intercept #DB in order to maintain a correct DR6 guest value.
4199 */
4200#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4201 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4202 && !CPUMIsGuestDebugStateActive(pVCpu))
4203#else
4204 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4205#endif
4206 {
4207 fInterceptMovDRx = true;
4208 fInterceptDB = true;
4209 }
4210
4211 /* Update guest DR7. */
4212 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4213 AssertRCReturn(rc, rc);
4214
4215 pVCpu->hm.s.fUsingHyperDR7 = false;
4216 }
4217
4218 /*
4219 * Update the exception bitmap regarding intercepting #DB generated by the guest.
4220 */
4221 if ( fInterceptDB
4222 || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4223 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
4224 else
4225 {
4226#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4227 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
4228#endif
4229 }
4230 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
4231 AssertRCReturn(rc, rc);
4232
4233 /*
4234 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4235 */
4236 if (fInterceptMovDRx)
4237 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4238 else
4239 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4240 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4241 AssertRCReturn(rc, rc);
4242
4243 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4244 return VINF_SUCCESS;
4245}
4246
4247
4248#ifdef VBOX_STRICT
4249/**
4250 * Strict function to validate segment registers.
4251 *
4252 * @remarks ASSUMES CR0 is up to date.
4253 */
4254static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4255{
4256 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4257 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4258 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4259 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4260 && ( !CPUMIsGuestInRealModeEx(pCtx)
4261 && !CPUMIsGuestInV86ModeEx(pCtx)))
4262 {
4263 /* Protected mode checks */
4264 /* CS */
4265 Assert(pCtx->cs.Attr.n.u1Present);
4266 Assert(!(pCtx->cs.Attr.u & 0xf00));
4267 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4268 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4269 || !(pCtx->cs.Attr.n.u1Granularity));
4270 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4271 || (pCtx->cs.Attr.n.u1Granularity));
4272 /* CS cannot be loaded with NULL in protected mode. */
4273 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4274 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4275 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4276 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4277 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4278 else
4279 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4280 /* SS */
4281 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4282 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4283 if ( !(pCtx->cr0 & X86_CR0_PE)
4284 || pCtx->cs.Attr.n.u4Type == 3)
4285 {
4286 Assert(!pCtx->ss.Attr.n.u2Dpl);
4287 }
4288 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4289 {
4290 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4291 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4292 Assert(pCtx->ss.Attr.n.u1Present);
4293 Assert(!(pCtx->ss.Attr.u & 0xf00));
4294 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4295 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4296 || !(pCtx->ss.Attr.n.u1Granularity));
4297 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4298 || (pCtx->ss.Attr.n.u1Granularity));
4299 }
4300 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4301 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4302 {
4303 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4304 Assert(pCtx->ds.Attr.n.u1Present);
4305 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4306 Assert(!(pCtx->ds.Attr.u & 0xf00));
4307 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4308 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4309 || !(pCtx->ds.Attr.n.u1Granularity));
4310 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4311 || (pCtx->ds.Attr.n.u1Granularity));
4312 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4313 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4314 }
4315 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4316 {
4317 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4318 Assert(pCtx->es.Attr.n.u1Present);
4319 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4320 Assert(!(pCtx->es.Attr.u & 0xf00));
4321 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4322 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4323 || !(pCtx->es.Attr.n.u1Granularity));
4324 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4325 || (pCtx->es.Attr.n.u1Granularity));
4326 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4327 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4328 }
4329 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4330 {
4331 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4332 Assert(pCtx->fs.Attr.n.u1Present);
4333 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4334 Assert(!(pCtx->fs.Attr.u & 0xf00));
4335 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4336 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4337 || !(pCtx->fs.Attr.n.u1Granularity));
4338 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4339 || (pCtx->fs.Attr.n.u1Granularity));
4340 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4341 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4342 }
4343 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4344 {
4345 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4346 Assert(pCtx->gs.Attr.n.u1Present);
4347 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4348 Assert(!(pCtx->gs.Attr.u & 0xf00));
4349 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4350 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4351 || !(pCtx->gs.Attr.n.u1Granularity));
4352 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4353 || (pCtx->gs.Attr.n.u1Granularity));
4354 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4355 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4356 }
4357 /* 64-bit capable CPUs. */
4358# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4359 if (HMVMX_IS_64BIT_HOST_MODE())
4360 {
4361 Assert(!(pCtx->cs.u64Base >> 32));
4362 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4363 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4364 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4365 }
4366# endif
4367 }
4368 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4369 || ( CPUMIsGuestInRealModeEx(pCtx)
4370 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4371 {
4372 /* Real and v86 mode checks. */
4373 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4374 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4375 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4376 {
4377 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4378 }
4379 else
4380 {
4381 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4382 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4383 }
4384
4385 /* CS */
4386 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4387 Assert(pCtx->cs.u32Limit == 0xffff);
4388 Assert(u32CSAttr == 0xf3);
4389 /* SS */
4390 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4391 Assert(pCtx->ss.u32Limit == 0xffff);
4392 Assert(u32SSAttr == 0xf3);
4393 /* DS */
4394 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4395 Assert(pCtx->ds.u32Limit == 0xffff);
4396 Assert(u32DSAttr == 0xf3);
4397 /* ES */
4398 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4399 Assert(pCtx->es.u32Limit == 0xffff);
4400 Assert(u32ESAttr == 0xf3);
4401 /* FS */
4402 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4403 Assert(pCtx->fs.u32Limit == 0xffff);
4404 Assert(u32FSAttr == 0xf3);
4405 /* GS */
4406 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4407 Assert(pCtx->gs.u32Limit == 0xffff);
4408 Assert(u32GSAttr == 0xf3);
4409 /* 64-bit capable CPUs. */
4410# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4411 if (HMVMX_IS_64BIT_HOST_MODE())
4412 {
4413 Assert(!(pCtx->cs.u64Base >> 32));
4414 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4415 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4416 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4417 }
4418# endif
4419 }
4420}
4421#endif /* VBOX_STRICT */
4422
4423
4424/**
4425 * Writes a guest segment register into the guest-state area in the VMCS.
4426 *
4427 * @returns VBox status code.
4428 * @param pVCpu Pointer to the VMCPU.
4429 * @param idxSel Index of the selector in the VMCS.
4430 * @param idxLimit Index of the segment limit in the VMCS.
4431 * @param idxBase Index of the segment base in the VMCS.
4432 * @param idxAccess Index of the access rights of the segment in the VMCS.
4433 * @param pSelReg Pointer to the segment selector.
4434 *
4435 * @remarks No-long-jump zone!!!
4436 */
4437static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4438 uint32_t idxAccess, PCPUMSELREG pSelReg)
4439{
4440 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4441 AssertRCReturn(rc, rc);
4442 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4443 AssertRCReturn(rc, rc);
4444 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4445 AssertRCReturn(rc, rc);
4446
4447 uint32_t u32Access = pSelReg->Attr.u;
4448 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4449 {
4450 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4451 u32Access = 0xf3;
4452 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4453 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4454 }
4455 else
4456 {
4457 /*
4458 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4459 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4460 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4461 * loaded in protected-mode have their attribute as 0.
4462 */
4463 if (!u32Access)
4464 u32Access = X86DESCATTR_UNUSABLE;
4465 }
4466
4467 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4468 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4469 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4470
4471 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4472 AssertRCReturn(rc, rc);
4473 return rc;
4474}
4475
4476
4477/**
4478 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4479 * into the guest-state area in the VMCS.
4480 *
4481 * @returns VBox status code.
4482 * @param pVM Pointer to the VM.
4483 * @param pVCPU Pointer to the VMCPU.
4484 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4485 * out-of-sync. Make sure to update the required fields
4486 * before using them.
4487 *
4488 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4489 * @remarks No-long-jump zone!!!
4490 */
4491static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4492{
4493 int rc = VERR_INTERNAL_ERROR_5;
4494 PVM pVM = pVCpu->CTX_SUFF(pVM);
4495
4496 /*
4497 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4498 */
4499 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4500 {
4501 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4502 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4503 {
4504 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4505 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4506 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4507 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4508 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4509 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4510 }
4511
4512#ifdef VBOX_WITH_REM
4513 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4514 {
4515 Assert(pVM->hm.s.vmx.pRealModeTSS);
4516 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4517 if ( pVCpu->hm.s.vmx.fWasInRealMode
4518 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4519 {
4520 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4521 in real-mode (e.g. OpenBSD 4.0) */
4522 REMFlushTBs(pVM);
4523 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4524 pVCpu->hm.s.vmx.fWasInRealMode = false;
4525 }
4526 }
4527#endif
4528 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4529 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4530 AssertRCReturn(rc, rc);
4531 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4532 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4533 AssertRCReturn(rc, rc);
4534 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4535 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4536 AssertRCReturn(rc, rc);
4537 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4538 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4539 AssertRCReturn(rc, rc);
4540 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4541 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4542 AssertRCReturn(rc, rc);
4543 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4544 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4545 AssertRCReturn(rc, rc);
4546
4547#ifdef VBOX_STRICT
4548 /* Validate. */
4549 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4550#endif
4551
4552 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4553 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4554 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4555 }
4556
4557 /*
4558 * Guest TR.
4559 */
4560 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4561 {
4562 /*
4563 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4564 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4565 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4566 */
4567 uint16_t u16Sel = 0;
4568 uint32_t u32Limit = 0;
4569 uint64_t u64Base = 0;
4570 uint32_t u32AccessRights = 0;
4571
4572 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4573 {
4574 u16Sel = pMixedCtx->tr.Sel;
4575 u32Limit = pMixedCtx->tr.u32Limit;
4576 u64Base = pMixedCtx->tr.u64Base;
4577 u32AccessRights = pMixedCtx->tr.Attr.u;
4578 }
4579 else
4580 {
4581 Assert(pVM->hm.s.vmx.pRealModeTSS);
4582 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4583
4584 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4585 RTGCPHYS GCPhys;
4586 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4587 AssertRCReturn(rc, rc);
4588
4589 X86DESCATTR DescAttr;
4590 DescAttr.u = 0;
4591 DescAttr.n.u1Present = 1;
4592 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4593
4594 u16Sel = 0;
4595 u32Limit = HM_VTX_TSS_SIZE;
4596 u64Base = GCPhys; /* in real-mode phys = virt. */
4597 u32AccessRights = DescAttr.u;
4598 }
4599
4600 /* Validate. */
4601 Assert(!(u16Sel & RT_BIT(2)));
4602 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4603 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4604 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4605 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4606 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4607 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4608 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4609 Assert( (u32Limit & 0xfff) == 0xfff
4610 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4611 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4612 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4613
4614 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4615 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4616 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4617 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4618
4619 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4620 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4621 }
4622
4623 /*
4624 * Guest GDTR.
4625 */
4626 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4627 {
4628 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4629 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4630
4631 /* Validate. */
4632 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4633
4634 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4635 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4636 }
4637
4638 /*
4639 * Guest LDTR.
4640 */
4641 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4642 {
4643 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4644 uint32_t u32Access = 0;
4645 if (!pMixedCtx->ldtr.Attr.u)
4646 u32Access = X86DESCATTR_UNUSABLE;
4647 else
4648 u32Access = pMixedCtx->ldtr.Attr.u;
4649
4650 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4651 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4652 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4653 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4654
4655 /* Validate. */
4656 if (!(u32Access & X86DESCATTR_UNUSABLE))
4657 {
4658 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4659 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4660 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4661 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4662 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4663 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4664 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4665 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4666 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4667 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4668 }
4669
4670 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4671 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4672 }
4673
4674 /*
4675 * Guest IDTR.
4676 */
4677 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4678 {
4679 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4680 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4681
4682 /* Validate. */
4683 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4684
4685 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4686 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4687 }
4688
4689 return VINF_SUCCESS;
4690}
4691
4692
4693/**
4694 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4695 * areas.
4696 *
4697 * These MSRs will automatically be loaded to the host CPU on every successful
4698 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4699 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4700 * -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4701 *
4702 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4703 *
4704 * @returns VBox status code.
4705 * @param pVCpu Pointer to the VMCPU.
4706 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4707 * out-of-sync. Make sure to update the required fields
4708 * before using them.
4709 *
4710 * @remarks No-long-jump zone!!!
4711 */
4712static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4713{
4714 AssertPtr(pVCpu);
4715 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4716
4717 /*
4718 * MSRs that we use the auto-load/store MSR area in the VMCS.
4719 */
4720 PVM pVM = pVCpu->CTX_SUFF(pVM);
4721 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4722 {
4723 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4724#if HC_ARCH_BITS == 32 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4725 if (pVM->hm.s.fAllow64BitGuests)
4726 {
4727 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4728 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4729 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4730 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4731# ifdef DEBUG
4732 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4733 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4734 {
4735 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4736 pMsr->u64Value));
4737 }
4738# endif
4739 }
4740#endif
4741 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4742 }
4743
4744 /*
4745 * Guest Sysenter MSRs.
4746 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4747 * VM-exits on WRMSRs for these MSRs.
4748 */
4749 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4750 {
4751 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4752 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4753 }
4754
4755 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4756 {
4757 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4758 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4759 }
4760
4761 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4762 {
4763 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4764 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4765 }
4766
4767 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4768 {
4769 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4770 {
4771 /*
4772 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4773 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4774 */
4775 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4776 {
4777 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4778 AssertRCReturn(rc,rc);
4779 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4780 }
4781 else
4782 {
4783 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */);
4784 /* We need to intercept reads too, see @bugref{7386} comment #16. */
4785 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4786 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4787 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4788 }
4789 }
4790 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4791 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4792 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4793 }
4794
4795 return VINF_SUCCESS;
4796}
4797
4798
4799/**
4800 * Loads the guest activity state into the guest-state area in the VMCS.
4801 *
4802 * @returns VBox status code.
4803 * @param pVCpu Pointer to the VMCPU.
4804 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4805 * out-of-sync. Make sure to update the required fields
4806 * before using them.
4807 *
4808 * @remarks No-long-jump zone!!!
4809 */
4810static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4811{
4812 NOREF(pCtx);
4813 /** @todo See if we can make use of other states, e.g.
4814 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4815 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4816 {
4817 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4818 AssertRCReturn(rc, rc);
4819
4820 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4821 }
4822 return VINF_SUCCESS;
4823}
4824
4825
4826/**
4827 * Sets up the appropriate function to run guest code.
4828 *
4829 * @returns VBox status code.
4830 * @param pVCpu Pointer to the VMCPU.
4831 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4832 * out-of-sync. Make sure to update the required fields
4833 * before using them.
4834 *
4835 * @remarks No-long-jump zone!!!
4836 */
4837static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4838{
4839 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4840 {
4841#ifndef VBOX_ENABLE_64_BITS_GUESTS
4842 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4843#endif
4844 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4845#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4846 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4847 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4848 {
4849 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4850 {
4851 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4852 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4853 | HM_CHANGED_VMX_ENTRY_CTLS
4854 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4855 }
4856 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4857 }
4858#else
4859 /* 64-bit host or hybrid host. */
4860 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4861#endif
4862 }
4863 else
4864 {
4865 /* Guest is not in long mode, use the 32-bit handler. */
4866#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4867 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4868 {
4869 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4870 {
4871 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4872 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4873 | HM_CHANGED_VMX_ENTRY_CTLS
4874 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4875 }
4876 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4877 }
4878#else
4879 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4880#endif
4881 }
4882 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4883 return VINF_SUCCESS;
4884}
4885
4886
4887/**
4888 * Wrapper for running the guest code in VT-x.
4889 *
4890 * @returns VBox strict status code.
4891 * @param pVM Pointer to the VM.
4892 * @param pVCpu Pointer to the VMCPU.
4893 * @param pCtx Pointer to the guest-CPU context.
4894 *
4895 * @remarks No-long-jump zone!!!
4896 */
4897DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4898{
4899 /*
4900 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4901 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4902 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4903 */
4904 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4905 /** @todo Add stats for resume vs launch. */
4906#ifdef VBOX_WITH_KERNEL_USING_XMM
4907 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4908#else
4909 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4910#endif
4911}
4912
4913
4914/**
4915 * Reports world-switch error and dumps some useful debug info.
4916 *
4917 * @param pVM Pointer to the VM.
4918 * @param pVCpu Pointer to the VMCPU.
4919 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4920 * @param pCtx Pointer to the guest-CPU context.
4921 * @param pVmxTransient Pointer to the VMX transient structure (only
4922 * exitReason updated).
4923 */
4924static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4925{
4926 Assert(pVM);
4927 Assert(pVCpu);
4928 Assert(pCtx);
4929 Assert(pVmxTransient);
4930 HMVMX_ASSERT_PREEMPT_SAFE();
4931
4932 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4933 switch (rcVMRun)
4934 {
4935 case VERR_VMX_INVALID_VMXON_PTR:
4936 AssertFailed();
4937 break;
4938 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4939 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4940 {
4941 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4942 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4943 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4944 AssertRC(rc);
4945
4946 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4947 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4948 Cannot do it here as we may have been long preempted. */
4949
4950#ifdef VBOX_STRICT
4951 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4952 pVmxTransient->uExitReason));
4953 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4954 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4955 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4956 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4957 else
4958 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4959 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4960 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4961
4962 /* VMX control bits. */
4963 uint32_t u32Val;
4964 uint64_t u64Val;
4965 HMVMXHCUINTREG uHCReg;
4966 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4967 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4968 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4969 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4970 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4971 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4972 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4973 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4974 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4975 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4976 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4977 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4978 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4979 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4980 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4981 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4982 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4983 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4984 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4985 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4986 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4987 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4988 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4989 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4990 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4991 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4992 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4993 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4994 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4995 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4996 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4997 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4998 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4999 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5000 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5001 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5002 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5003 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5004 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5005 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5006 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5007 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5008
5009 /* Guest bits. */
5010 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5011 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5012 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5013 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5014 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5015 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5016 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
5017 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
5018
5019 /* Host bits. */
5020 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5021 Log4(("Host CR0 %#RHr\n", uHCReg));
5022 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5023 Log4(("Host CR3 %#RHr\n", uHCReg));
5024 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5025 Log4(("Host CR4 %#RHr\n", uHCReg));
5026
5027 RTGDTR HostGdtr;
5028 PCX86DESCHC pDesc;
5029 ASMGetGDTR(&HostGdtr);
5030 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
5031 Log4(("Host CS %#08x\n", u32Val));
5032 if (u32Val < HostGdtr.cbGdt)
5033 {
5034 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5035 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5036 }
5037
5038 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
5039 Log4(("Host DS %#08x\n", u32Val));
5040 if (u32Val < HostGdtr.cbGdt)
5041 {
5042 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5043 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5044 }
5045
5046 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
5047 Log4(("Host ES %#08x\n", u32Val));
5048 if (u32Val < HostGdtr.cbGdt)
5049 {
5050 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5051 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5052 }
5053
5054 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
5055 Log4(("Host FS %#08x\n", u32Val));
5056 if (u32Val < HostGdtr.cbGdt)
5057 {
5058 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5059 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5060 }
5061
5062 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
5063 Log4(("Host GS %#08x\n", u32Val));
5064 if (u32Val < HostGdtr.cbGdt)
5065 {
5066 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5067 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5068 }
5069
5070 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
5071 Log4(("Host SS %#08x\n", u32Val));
5072 if (u32Val < HostGdtr.cbGdt)
5073 {
5074 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5075 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5076 }
5077
5078 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
5079 Log4(("Host TR %#08x\n", u32Val));
5080 if (u32Val < HostGdtr.cbGdt)
5081 {
5082 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5083 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5084 }
5085
5086 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5087 Log4(("Host TR Base %#RHv\n", uHCReg));
5088 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5089 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5090 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5091 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5092 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5093 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5094 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5095 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5096 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5097 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5098 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5099 Log4(("Host RSP %#RHv\n", uHCReg));
5100 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5101 Log4(("Host RIP %#RHv\n", uHCReg));
5102# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5103 if (HMVMX_IS_64BIT_HOST_MODE())
5104 {
5105 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5106 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5107 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5108 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5109 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5110 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5111 }
5112# endif
5113#endif /* VBOX_STRICT */
5114 break;
5115 }
5116
5117 default:
5118 /* Impossible */
5119 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5120 break;
5121 }
5122 NOREF(pVM); NOREF(pCtx);
5123}
5124
5125
5126#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5127#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5128# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5129#endif
5130#ifdef VBOX_STRICT
5131static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5132{
5133 switch (idxField)
5134 {
5135 case VMX_VMCS_GUEST_RIP:
5136 case VMX_VMCS_GUEST_RSP:
5137 case VMX_VMCS_GUEST_SYSENTER_EIP:
5138 case VMX_VMCS_GUEST_SYSENTER_ESP:
5139 case VMX_VMCS_GUEST_GDTR_BASE:
5140 case VMX_VMCS_GUEST_IDTR_BASE:
5141 case VMX_VMCS_GUEST_CS_BASE:
5142 case VMX_VMCS_GUEST_DS_BASE:
5143 case VMX_VMCS_GUEST_ES_BASE:
5144 case VMX_VMCS_GUEST_FS_BASE:
5145 case VMX_VMCS_GUEST_GS_BASE:
5146 case VMX_VMCS_GUEST_SS_BASE:
5147 case VMX_VMCS_GUEST_LDTR_BASE:
5148 case VMX_VMCS_GUEST_TR_BASE:
5149 case VMX_VMCS_GUEST_CR3:
5150 return true;
5151 }
5152 return false;
5153}
5154
5155static bool hmR0VmxIsValidReadField(uint32_t idxField)
5156{
5157 switch (idxField)
5158 {
5159 /* Read-only fields. */
5160 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5161 return true;
5162 }
5163 /* Remaining readable fields should also be writable. */
5164 return hmR0VmxIsValidWriteField(idxField);
5165}
5166#endif /* VBOX_STRICT */
5167
5168
5169/**
5170 * Executes the specified handler in 64-bit mode.
5171 *
5172 * @returns VBox status code.
5173 * @param pVM Pointer to the VM.
5174 * @param pVCpu Pointer to the VMCPU.
5175 * @param pCtx Pointer to the guest CPU context.
5176 * @param enmOp The operation to perform.
5177 * @param cbParam Number of parameters.
5178 * @param paParam Array of 32-bit parameters.
5179 */
5180VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
5181 uint32_t *paParam)
5182{
5183 int rc, rc2;
5184 PHMGLOBALCPUINFO pCpu;
5185 RTHCPHYS HCPhysCpuPage;
5186 RTCCUINTREG uOldEflags;
5187
5188 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5189 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5190 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5191 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5192
5193#ifdef VBOX_STRICT
5194 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5195 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5196
5197 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5198 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5199#endif
5200
5201 /* Disable interrupts. */
5202 uOldEflags = ASMIntDisableFlags();
5203
5204#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5205 RTCPUID idHostCpu = RTMpCpuId();
5206 CPUMR0SetLApic(pVCpu, idHostCpu);
5207#endif
5208
5209 pCpu = HMR0GetCurrentCpu();
5210 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5211
5212 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5213 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5214
5215 /* Leave VMX Root Mode. */
5216 VMXDisable();
5217
5218 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5219
5220 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5221 CPUMSetHyperEIP(pVCpu, enmOp);
5222 for (int i = (int)cbParam - 1; i >= 0; i--)
5223 CPUMPushHyper(pVCpu, paParam[i]);
5224
5225 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5226
5227 /* Call the switcher. */
5228 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5229 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5230
5231 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5232 /* Make sure the VMX instructions don't cause #UD faults. */
5233 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
5234
5235 /* Re-enter VMX Root Mode */
5236 rc2 = VMXEnable(HCPhysCpuPage);
5237 if (RT_FAILURE(rc2))
5238 {
5239 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5240 ASMSetFlags(uOldEflags);
5241 return rc2;
5242 }
5243
5244 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5245 AssertRC(rc2);
5246 Assert(!(ASMGetFlags() & X86_EFL_IF));
5247 ASMSetFlags(uOldEflags);
5248 return rc;
5249}
5250
5251
5252/**
5253 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5254 * supporting 64-bit guests.
5255 *
5256 * @returns VBox status code.
5257 * @param fResume Whether to VMLAUNCH or VMRESUME.
5258 * @param pCtx Pointer to the guest-CPU context.
5259 * @param pCache Pointer to the VMCS cache.
5260 * @param pVM Pointer to the VM.
5261 * @param pVCpu Pointer to the VMCPU.
5262 */
5263DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5264{
5265 uint32_t aParam[6];
5266 PHMGLOBALCPUINFO pCpu = NULL;
5267 RTHCPHYS HCPhysCpuPage = 0;
5268 int rc = VERR_INTERNAL_ERROR_5;
5269
5270 pCpu = HMR0GetCurrentCpu();
5271 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5272
5273#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5274 pCache->uPos = 1;
5275 pCache->interPD = PGMGetInterPaeCR3(pVM);
5276 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5277#endif
5278
5279#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5280 pCache->TestIn.HCPhysCpuPage = 0;
5281 pCache->TestIn.HCPhysVmcs = 0;
5282 pCache->TestIn.pCache = 0;
5283 pCache->TestOut.HCPhysVmcs = 0;
5284 pCache->TestOut.pCache = 0;
5285 pCache->TestOut.pCtx = 0;
5286 pCache->TestOut.eflags = 0;
5287#endif
5288
5289 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5290 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5291 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5292 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5293 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5294 aParam[5] = 0;
5295
5296#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5297 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5298 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5299#endif
5300 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
5301
5302#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5303 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5304 Assert(pCtx->dr[4] == 10);
5305 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5306#endif
5307
5308#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5309 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5310 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5311 pVCpu->hm.s.vmx.HCPhysVmcs));
5312 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5313 pCache->TestOut.HCPhysVmcs));
5314 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5315 pCache->TestOut.pCache));
5316 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5317 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5318 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5319 pCache->TestOut.pCtx));
5320 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5321#endif
5322 return rc;
5323}
5324
5325
5326/**
5327 * Initialize the VMCS-Read cache.
5328 *
5329 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5330 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5331 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5332 * (those that have a 32-bit FULL & HIGH part).
5333 *
5334 * @returns VBox status code.
5335 * @param pVM Pointer to the VM.
5336 * @param pVCpu Pointer to the VMCPU.
5337 */
5338static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5339{
5340#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5341{ \
5342 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5343 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5344 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5345 ++cReadFields; \
5346}
5347
5348 AssertPtr(pVM);
5349 AssertPtr(pVCpu);
5350 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5351 uint32_t cReadFields = 0;
5352
5353 /*
5354 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5355 * and serve to indicate exceptions to the rules.
5356 */
5357
5358 /* Guest-natural selector base fields. */
5359#if 0
5360 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5361 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5362 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5363#endif
5364 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5365 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5366 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5367 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5368 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5369 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5370 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5371 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5372 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5373 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5374 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5375 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5376#if 0
5377 /* Unused natural width guest-state fields. */
5378 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5379 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5380#endif
5381 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5382 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5383
5384 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5385#if 0
5386 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5387 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5388 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5389 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5390 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5391 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5392 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5393 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5394 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5395#endif
5396
5397 /* Natural width guest-state fields. */
5398 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5399#if 0
5400 /* Currently unused field. */
5401 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5402#endif
5403
5404 if (pVM->hm.s.fNestedPaging)
5405 {
5406 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5407 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5408 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5409 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5410 }
5411 else
5412 {
5413 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5414 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5415 }
5416
5417#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5418 return VINF_SUCCESS;
5419}
5420
5421
5422/**
5423 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5424 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5425 * darwin, running 64-bit guests).
5426 *
5427 * @returns VBox status code.
5428 * @param pVCpu Pointer to the VMCPU.
5429 * @param idxField The VMCS field encoding.
5430 * @param u64Val 16, 32 or 64-bit value.
5431 */
5432VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5433{
5434 int rc;
5435 switch (idxField)
5436 {
5437 /*
5438 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5439 */
5440 /* 64-bit Control fields. */
5441 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5442 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5443 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5444 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5445 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5446 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5447 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5448 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5449 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5450 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5451 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5452 case VMX_VMCS64_CTRL_EPTP_FULL:
5453 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5454 /* 64-bit Guest-state fields. */
5455 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5456 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5457 case VMX_VMCS64_GUEST_PAT_FULL:
5458 case VMX_VMCS64_GUEST_EFER_FULL:
5459 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5460 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5461 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5462 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5463 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5464 /* 64-bit Host-state fields. */
5465 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5466 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5467 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5468 {
5469 rc = VMXWriteVmcs32(idxField, u64Val);
5470 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5471 break;
5472 }
5473
5474 /*
5475 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5476 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5477 */
5478 /* Natural-width Guest-state fields. */
5479 case VMX_VMCS_GUEST_CR3:
5480 case VMX_VMCS_GUEST_ES_BASE:
5481 case VMX_VMCS_GUEST_CS_BASE:
5482 case VMX_VMCS_GUEST_SS_BASE:
5483 case VMX_VMCS_GUEST_DS_BASE:
5484 case VMX_VMCS_GUEST_FS_BASE:
5485 case VMX_VMCS_GUEST_GS_BASE:
5486 case VMX_VMCS_GUEST_LDTR_BASE:
5487 case VMX_VMCS_GUEST_TR_BASE:
5488 case VMX_VMCS_GUEST_GDTR_BASE:
5489 case VMX_VMCS_GUEST_IDTR_BASE:
5490 case VMX_VMCS_GUEST_RSP:
5491 case VMX_VMCS_GUEST_RIP:
5492 case VMX_VMCS_GUEST_SYSENTER_ESP:
5493 case VMX_VMCS_GUEST_SYSENTER_EIP:
5494 {
5495 if (!(u64Val >> 32))
5496 {
5497 /* If this field is 64-bit, VT-x will zero out the top bits. */
5498 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5499 }
5500 else
5501 {
5502 /* Assert that only the 32->64 switcher case should ever come here. */
5503 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5504 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5505 }
5506 break;
5507 }
5508
5509 default:
5510 {
5511 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5512 rc = VERR_INVALID_PARAMETER;
5513 break;
5514 }
5515 }
5516 AssertRCReturn(rc, rc);
5517 return rc;
5518}
5519
5520
5521/**
5522 * Queue up a VMWRITE by using the VMCS write cache.
5523 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5524 *
5525 * @param pVCpu Pointer to the VMCPU.
5526 * @param idxField The VMCS field encoding.
5527 * @param u64Val 16, 32 or 64-bit value.
5528 */
5529VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5530{
5531 AssertPtr(pVCpu);
5532 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5533
5534 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5535 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5536
5537 /* Make sure there are no duplicates. */
5538 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5539 {
5540 if (pCache->Write.aField[i] == idxField)
5541 {
5542 pCache->Write.aFieldVal[i] = u64Val;
5543 return VINF_SUCCESS;
5544 }
5545 }
5546
5547 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5548 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5549 pCache->Write.cValidEntries++;
5550 return VINF_SUCCESS;
5551}
5552
5553/* Enable later when the assembly code uses these as callbacks. */
5554#if 0
5555/*
5556 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5557 *
5558 * @param pVCpu Pointer to the VMCPU.
5559 * @param pCache Pointer to the VMCS cache.
5560 *
5561 * @remarks No-long-jump zone!!!
5562 */
5563VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5564{
5565 AssertPtr(pCache);
5566 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5567 {
5568 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5569 AssertRC(rc);
5570 }
5571 pCache->Write.cValidEntries = 0;
5572}
5573
5574
5575/**
5576 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
5577 *
5578 * @param pVCpu Pointer to the VMCPU.
5579 * @param pCache Pointer to the VMCS cache.
5580 *
5581 * @remarks No-long-jump zone!!!
5582 */
5583VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5584{
5585 AssertPtr(pCache);
5586 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5587 {
5588 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5589 AssertRC(rc);
5590 }
5591}
5592#endif
5593#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5594
5595
5596/**
5597 * Sets up the usage of TSC-offsetting and updates the VMCS.
5598 *
5599 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5600 * VMX preemption timer.
5601 *
5602 * @returns VBox status code.
5603 * @param pVM Pointer to the cross context VM structure.
5604 * @param pVCpu Pointer to the VMCPU.
5605 *
5606 * @remarks No-long-jump zone!!!
5607 */
5608static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5609{
5610 int rc;
5611 bool fOffsettedTsc;
5612 bool fParavirtTsc;
5613 if (pVM->hm.s.vmx.fUsePreemptTimer)
5614 {
5615 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5616 &fOffsettedTsc, &fParavirtTsc);
5617
5618 /* Make sure the returned values have sane upper and lower boundaries. */
5619 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5620 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5621 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5622 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5623
5624 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5625 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5626 }
5627 else
5628 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5629
5630 /** @todo later optimize this to be done elsewhere and not before every
5631 * VM-entry. */
5632 if (fParavirtTsc)
5633 {
5634 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5635 AssertRC(rc);
5636 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5637 }
5638
5639 if (fOffsettedTsc)
5640 {
5641 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5642 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5643
5644 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5645 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5646 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5647 }
5648 else
5649 {
5650 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5651 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5652 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5653 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5654 }
5655}
5656
5657
5658/**
5659 * Determines if an exception is a contributory exception.
5660 *
5661 * Contributory exceptions are ones which can cause double-faults unless the
5662 * original exception was a benign exception. Page-fault is intentionally not
5663 * included here as it's a conditional contributory exception.
5664 *
5665 * @returns true if the exception is contributory, false otherwise.
5666 * @param uVector The exception vector.
5667 */
5668DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5669{
5670 switch (uVector)
5671 {
5672 case X86_XCPT_GP:
5673 case X86_XCPT_SS:
5674 case X86_XCPT_NP:
5675 case X86_XCPT_TS:
5676 case X86_XCPT_DE:
5677 return true;
5678 default:
5679 break;
5680 }
5681 return false;
5682}
5683
5684
5685/**
5686 * Sets an event as a pending event to be injected into the guest.
5687 *
5688 * @param pVCpu Pointer to the VMCPU.
5689 * @param u32IntInfo The VM-entry interruption-information field.
5690 * @param cbInstr The VM-entry instruction length in bytes (for software
5691 * interrupts, exceptions and privileged software
5692 * exceptions).
5693 * @param u32ErrCode The VM-entry exception error code.
5694 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5695 * page-fault.
5696 *
5697 * @remarks Statistics counter assumes this is a guest event being injected or
5698 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5699 * always incremented.
5700 */
5701DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5702 RTGCUINTPTR GCPtrFaultAddress)
5703{
5704 Assert(!pVCpu->hm.s.Event.fPending);
5705 pVCpu->hm.s.Event.fPending = true;
5706 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5707 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5708 pVCpu->hm.s.Event.cbInstr = cbInstr;
5709 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5710
5711 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5712}
5713
5714
5715/**
5716 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5717 *
5718 * @param pVCpu Pointer to the VMCPU.
5719 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5720 * out-of-sync. Make sure to update the required fields
5721 * before using them.
5722 */
5723DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5724{
5725 NOREF(pMixedCtx);
5726 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5727 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5728 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5729 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5730}
5731
5732
5733/**
5734 * Handle a condition that occurred while delivering an event through the guest
5735 * IDT.
5736 *
5737 * @returns VBox status code (informational error codes included).
5738 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5739 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5740 * continue execution of the guest which will delivery the #DF.
5741 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5742 *
5743 * @param pVCpu Pointer to the VMCPU.
5744 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5745 * out-of-sync. Make sure to update the required fields
5746 * before using them.
5747 * @param pVmxTransient Pointer to the VMX transient structure.
5748 *
5749 * @remarks No-long-jump zone!!!
5750 */
5751static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5752{
5753 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5754
5755 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5756 AssertRCReturn(rc, rc);
5757 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5758 AssertRCReturn(rc, rc);
5759
5760 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5761 {
5762 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5763 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5764
5765 typedef enum
5766 {
5767 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5768 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5769 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5770 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5771 } VMXREFLECTXCPT;
5772
5773 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5774 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5775 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5776 {
5777 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5778 {
5779 enmReflect = VMXREFLECTXCPT_XCPT;
5780#ifdef VBOX_STRICT
5781 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5782 && uExitVector == X86_XCPT_PF)
5783 {
5784 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5785 }
5786#endif
5787 if ( uExitVector == X86_XCPT_PF
5788 && uIdtVector == X86_XCPT_PF)
5789 {
5790 pVmxTransient->fVectoringDoublePF = true;
5791 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5792 }
5793 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5794 && hmR0VmxIsContributoryXcpt(uExitVector)
5795 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5796 || uIdtVector == X86_XCPT_PF))
5797 {
5798 enmReflect = VMXREFLECTXCPT_DF;
5799 }
5800 else if (uIdtVector == X86_XCPT_DF)
5801 enmReflect = VMXREFLECTXCPT_TF;
5802 }
5803 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5804 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5805 {
5806 /*
5807 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5808 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5809 */
5810 enmReflect = VMXREFLECTXCPT_XCPT;
5811
5812 if (uExitVector == X86_XCPT_PF)
5813 {
5814 pVmxTransient->fVectoringPF = true;
5815 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5816 }
5817 }
5818 }
5819 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5820 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5821 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5822 {
5823 /*
5824 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5825 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5826 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5827 */
5828 enmReflect = VMXREFLECTXCPT_XCPT;
5829 }
5830
5831 /*
5832 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5833 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5834 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5835 *
5836 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5837 */
5838 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5839 && enmReflect == VMXREFLECTXCPT_XCPT
5840 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5841 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5842 {
5843 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5844 }
5845
5846 switch (enmReflect)
5847 {
5848 case VMXREFLECTXCPT_XCPT:
5849 {
5850 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5851 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5852 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5853
5854 uint32_t u32ErrCode = 0;
5855 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5856 {
5857 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5858 AssertRCReturn(rc, rc);
5859 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5860 }
5861
5862 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5863 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5864 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5865 rc = VINF_SUCCESS;
5866 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5867 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5868
5869 break;
5870 }
5871
5872 case VMXREFLECTXCPT_DF:
5873 {
5874 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5875 rc = VINF_HM_DOUBLE_FAULT;
5876 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5877 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5878
5879 break;
5880 }
5881
5882 case VMXREFLECTXCPT_TF:
5883 {
5884 rc = VINF_EM_RESET;
5885 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5886 uExitVector));
5887 break;
5888 }
5889
5890 default:
5891 Assert(rc == VINF_SUCCESS);
5892 break;
5893 }
5894 }
5895 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5896 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5897 && uExitVector != X86_XCPT_DF
5898 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5899 {
5900 /*
5901 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5902 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5903 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5904 */
5905 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5906 {
5907 Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
5908 pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5909 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5910 }
5911 }
5912
5913 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5914 return rc;
5915}
5916
5917
5918/**
5919 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5920 *
5921 * @returns VBox status code.
5922 * @param pVCpu Pointer to the VMCPU.
5923 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5924 * out-of-sync. Make sure to update the required fields
5925 * before using them.
5926 *
5927 * @remarks No-long-jump zone!!!
5928 */
5929static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5930{
5931 NOREF(pMixedCtx);
5932
5933 /*
5934 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5935 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5936 */
5937 VMMRZCallRing3Disable(pVCpu);
5938 HM_DISABLE_PREEMPT_IF_NEEDED();
5939
5940 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5941 {
5942 uint32_t uVal = 0;
5943 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5944 AssertRCReturn(rc, rc);
5945
5946 uint32_t uShadow = 0;
5947 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5948 AssertRCReturn(rc, rc);
5949
5950 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5951 CPUMSetGuestCR0(pVCpu, uVal);
5952 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5953 }
5954
5955 HM_RESTORE_PREEMPT_IF_NEEDED();
5956 VMMRZCallRing3Enable(pVCpu);
5957 return VINF_SUCCESS;
5958}
5959
5960
5961/**
5962 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5963 *
5964 * @returns VBox status code.
5965 * @param pVCpu Pointer to the VMCPU.
5966 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5967 * out-of-sync. Make sure to update the required fields
5968 * before using them.
5969 *
5970 * @remarks No-long-jump zone!!!
5971 */
5972static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5973{
5974 NOREF(pMixedCtx);
5975
5976 int rc = VINF_SUCCESS;
5977 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5978 {
5979 uint32_t uVal = 0;
5980 uint32_t uShadow = 0;
5981 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5982 AssertRCReturn(rc, rc);
5983 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5984 AssertRCReturn(rc, rc);
5985
5986 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5987 CPUMSetGuestCR4(pVCpu, uVal);
5988 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5989 }
5990 return rc;
5991}
5992
5993
5994/**
5995 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5996 *
5997 * @returns VBox status code.
5998 * @param pVCpu Pointer to the VMCPU.
5999 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6000 * out-of-sync. Make sure to update the required fields
6001 * before using them.
6002 *
6003 * @remarks No-long-jump zone!!!
6004 */
6005static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6006{
6007 int rc = VINF_SUCCESS;
6008 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6009 {
6010 uint64_t u64Val = 0;
6011 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6012 AssertRCReturn(rc, rc);
6013
6014 pMixedCtx->rip = u64Val;
6015 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6016 }
6017 return rc;
6018}
6019
6020
6021/**
6022 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6023 *
6024 * @returns VBox status code.
6025 * @param pVCpu Pointer to the VMCPU.
6026 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6027 * out-of-sync. Make sure to update the required fields
6028 * before using them.
6029 *
6030 * @remarks No-long-jump zone!!!
6031 */
6032static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6033{
6034 int rc = VINF_SUCCESS;
6035 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6036 {
6037 uint64_t u64Val = 0;
6038 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6039 AssertRCReturn(rc, rc);
6040
6041 pMixedCtx->rsp = u64Val;
6042 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6043 }
6044 return rc;
6045}
6046
6047
6048/**
6049 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6050 *
6051 * @returns VBox status code.
6052 * @param pVCpu Pointer to the VMCPU.
6053 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6054 * out-of-sync. Make sure to update the required fields
6055 * before using them.
6056 *
6057 * @remarks No-long-jump zone!!!
6058 */
6059static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6060{
6061 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6062 {
6063 uint32_t uVal = 0;
6064 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6065 AssertRCReturn(rc, rc);
6066
6067 pMixedCtx->eflags.u32 = uVal;
6068 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6069 {
6070 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6071 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6072
6073 pMixedCtx->eflags.Bits.u1VM = 0;
6074 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6075 }
6076
6077 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6078 }
6079 return VINF_SUCCESS;
6080}
6081
6082
6083/**
6084 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6085 * guest-CPU context.
6086 */
6087DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6088{
6089 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6090 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6091 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6092 return rc;
6093}
6094
6095
6096/**
6097 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6098 * from the guest-state area in the VMCS.
6099 *
6100 * @param pVCpu Pointer to the VMCPU.
6101 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6102 * out-of-sync. Make sure to update the required fields
6103 * before using them.
6104 *
6105 * @remarks No-long-jump zone!!!
6106 */
6107static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6108{
6109 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6110 {
6111 uint32_t uIntrState = 0;
6112 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6113 AssertRC(rc);
6114
6115 if (!uIntrState)
6116 {
6117 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6118 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6119
6120 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6121 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6122 }
6123 else
6124 {
6125 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6126 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6127 {
6128 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6129 AssertRC(rc);
6130 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6131 AssertRC(rc);
6132
6133 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6134 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6135 }
6136 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6137 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6138
6139 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6140 {
6141 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6142 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6143 }
6144 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6145 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6146 }
6147
6148 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6149 }
6150}
6151
6152
6153/**
6154 * Saves the guest's activity state.
6155 *
6156 * @returns VBox status code.
6157 * @param pVCpu Pointer to the VMCPU.
6158 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6159 * out-of-sync. Make sure to update the required fields
6160 * before using them.
6161 *
6162 * @remarks No-long-jump zone!!!
6163 */
6164static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6165{
6166 NOREF(pMixedCtx);
6167 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6168 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6169 return VINF_SUCCESS;
6170}
6171
6172
6173/**
6174 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6175 * the current VMCS into the guest-CPU context.
6176 *
6177 * @returns VBox status code.
6178 * @param pVCpu Pointer to the VMCPU.
6179 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6180 * out-of-sync. Make sure to update the required fields
6181 * before using them.
6182 *
6183 * @remarks No-long-jump zone!!!
6184 */
6185static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6186{
6187 int rc = VINF_SUCCESS;
6188 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6189 {
6190 uint32_t u32Val = 0;
6191 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6192 pMixedCtx->SysEnter.cs = u32Val;
6193 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6194 }
6195
6196 uint64_t u64Val = 0;
6197 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6198 {
6199 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6200 pMixedCtx->SysEnter.eip = u64Val;
6201 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6202 }
6203 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6204 {
6205 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6206 pMixedCtx->SysEnter.esp = u64Val;
6207 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6208 }
6209 return rc;
6210}
6211
6212
6213/**
6214 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6215 * the CPU back into the guest-CPU context.
6216 *
6217 * @returns VBox status code.
6218 * @param pVCpu Pointer to the VMCPU.
6219 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6220 * out-of-sync. Make sure to update the required fields
6221 * before using them.
6222 *
6223 * @remarks No-long-jump zone!!!
6224 */
6225static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6226{
6227#if HC_ARCH_BITS == 64
6228 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
6229 {
6230 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6231 VMMRZCallRing3Disable(pVCpu);
6232 HM_DISABLE_PREEMPT_IF_NEEDED();
6233
6234 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6235 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6236 {
6237 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6238 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6239 }
6240
6241 HM_RESTORE_PREEMPT_IF_NEEDED();
6242 VMMRZCallRing3Enable(pVCpu);
6243 }
6244 else
6245 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6246#else
6247 NOREF(pMixedCtx);
6248 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6249#endif
6250
6251 return VINF_SUCCESS;
6252}
6253
6254
6255/**
6256 * Saves the auto load/store'd guest MSRs from the current VMCS into
6257 * the guest-CPU context.
6258 *
6259 * @returns VBox status code.
6260 * @param pVCpu Pointer to the VMCPU.
6261 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6262 * out-of-sync. Make sure to update the required fields
6263 * before using them.
6264 *
6265 * @remarks No-long-jump zone!!!
6266 */
6267static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6268{
6269 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6270 return VINF_SUCCESS;
6271
6272 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6273 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6274 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6275 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6276 {
6277 switch (pMsr->u32Msr)
6278 {
6279 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6280 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6281 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6282 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6283 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6284 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6285 break;
6286
6287 default:
6288 {
6289 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6290 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6291 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6292 }
6293 }
6294 }
6295
6296 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6297 return VINF_SUCCESS;
6298}
6299
6300
6301/**
6302 * Saves the guest control registers from the current VMCS into the guest-CPU
6303 * context.
6304 *
6305 * @returns VBox status code.
6306 * @param pVCpu Pointer to the VMCPU.
6307 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6308 * out-of-sync. Make sure to update the required fields
6309 * before using them.
6310 *
6311 * @remarks No-long-jump zone!!!
6312 */
6313static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6314{
6315 /* Guest CR0. Guest FPU. */
6316 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6317 AssertRCReturn(rc, rc);
6318
6319 /* Guest CR4. */
6320 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6321 AssertRCReturn(rc, rc);
6322
6323 /* Guest CR2 - updated always during the world-switch or in #PF. */
6324 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6325 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6326 {
6327 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6328 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6329
6330 PVM pVM = pVCpu->CTX_SUFF(pVM);
6331 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6332 || ( pVM->hm.s.fNestedPaging
6333 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6334 {
6335 uint64_t u64Val = 0;
6336 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6337 if (pMixedCtx->cr3 != u64Val)
6338 {
6339 CPUMSetGuestCR3(pVCpu, u64Val);
6340 if (VMMRZCallRing3IsEnabled(pVCpu))
6341 {
6342 PGMUpdateCR3(pVCpu, u64Val);
6343 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6344 }
6345 else
6346 {
6347 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6348 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6349 }
6350 }
6351
6352 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6353 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6354 {
6355 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
6356 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
6357 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
6358 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
6359
6360 if (VMMRZCallRing3IsEnabled(pVCpu))
6361 {
6362 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6363 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6364 }
6365 else
6366 {
6367 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6368 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6369 }
6370 }
6371 }
6372
6373 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6374 }
6375
6376 /*
6377 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6378 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6379 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6380 *
6381 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6382 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6383 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6384 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6385 *
6386 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6387 */
6388 if (VMMRZCallRing3IsEnabled(pVCpu))
6389 {
6390 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6391 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6392
6393 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6394 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6395
6396 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6397 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6398 }
6399
6400 return rc;
6401}
6402
6403
6404/**
6405 * Reads a guest segment register from the current VMCS into the guest-CPU
6406 * context.
6407 *
6408 * @returns VBox status code.
6409 * @param pVCpu Pointer to the VMCPU.
6410 * @param idxSel Index of the selector in the VMCS.
6411 * @param idxLimit Index of the segment limit in the VMCS.
6412 * @param idxBase Index of the segment base in the VMCS.
6413 * @param idxAccess Index of the access rights of the segment in the VMCS.
6414 * @param pSelReg Pointer to the segment selector.
6415 *
6416 * @remarks No-long-jump zone!!!
6417 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6418 * macro as that takes care of whether to read from the VMCS cache or
6419 * not.
6420 */
6421DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6422 PCPUMSELREG pSelReg)
6423{
6424 NOREF(pVCpu);
6425
6426 uint32_t u32Val = 0;
6427 int rc = VMXReadVmcs32(idxSel, &u32Val);
6428 AssertRCReturn(rc, rc);
6429 pSelReg->Sel = (uint16_t)u32Val;
6430 pSelReg->ValidSel = (uint16_t)u32Val;
6431 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6432
6433 rc = VMXReadVmcs32(idxLimit, &u32Val);
6434 AssertRCReturn(rc, rc);
6435 pSelReg->u32Limit = u32Val;
6436
6437 uint64_t u64Val = 0;
6438 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6439 AssertRCReturn(rc, rc);
6440 pSelReg->u64Base = u64Val;
6441
6442 rc = VMXReadVmcs32(idxAccess, &u32Val);
6443 AssertRCReturn(rc, rc);
6444 pSelReg->Attr.u = u32Val;
6445
6446 /*
6447 * If VT-x marks the segment as unusable, most other bits remain undefined:
6448 * - For CS the L, D and G bits have meaning.
6449 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6450 * - For the remaining data segments no bits are defined.
6451 *
6452 * The present bit and the unusable bit has been observed to be set at the
6453 * same time (the selector was supposed to be invalid as we started executing
6454 * a V8086 interrupt in ring-0).
6455 *
6456 * What should be important for the rest of the VBox code, is that the P bit is
6457 * cleared. Some of the other VBox code recognizes the unusable bit, but
6458 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6459 * safe side here, we'll strip off P and other bits we don't care about. If
6460 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6461 *
6462 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6463 */
6464 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6465 {
6466 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6467
6468 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6469 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6470 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6471
6472 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6473#ifdef DEBUG_bird
6474 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6475 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6476 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6477#endif
6478 }
6479 return VINF_SUCCESS;
6480}
6481
6482
6483#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6484# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6485 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6486 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6487#else
6488# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6489 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6490 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6491#endif
6492
6493
6494/**
6495 * Saves the guest segment registers from the current VMCS into the guest-CPU
6496 * context.
6497 *
6498 * @returns VBox status code.
6499 * @param pVCpu Pointer to the VMCPU.
6500 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6501 * out-of-sync. Make sure to update the required fields
6502 * before using them.
6503 *
6504 * @remarks No-long-jump zone!!!
6505 */
6506static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6507{
6508 /* Guest segment registers. */
6509 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6510 {
6511 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6512 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6513 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6514 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6515 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6516 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6517 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6518
6519 /* Restore segment attributes for real-on-v86 mode hack. */
6520 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6521 {
6522 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6523 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6524 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6525 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6526 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6527 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6528 }
6529 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6530 }
6531
6532 return VINF_SUCCESS;
6533}
6534
6535
6536/**
6537 * Saves the guest descriptor table registers and task register from the current
6538 * VMCS into the guest-CPU context.
6539 *
6540 * @returns VBox status code.
6541 * @param pVCpu Pointer to the VMCPU.
6542 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6543 * out-of-sync. Make sure to update the required fields
6544 * before using them.
6545 *
6546 * @remarks No-long-jump zone!!!
6547 */
6548static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6549{
6550 int rc = VINF_SUCCESS;
6551
6552 /* Guest LDTR. */
6553 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6554 {
6555 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6556 AssertRCReturn(rc, rc);
6557 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6558 }
6559
6560 /* Guest GDTR. */
6561 uint64_t u64Val = 0;
6562 uint32_t u32Val = 0;
6563 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6564 {
6565 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6566 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6567 pMixedCtx->gdtr.pGdt = u64Val;
6568 pMixedCtx->gdtr.cbGdt = u32Val;
6569 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6570 }
6571
6572 /* Guest IDTR. */
6573 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6574 {
6575 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6576 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6577 pMixedCtx->idtr.pIdt = u64Val;
6578 pMixedCtx->idtr.cbIdt = u32Val;
6579 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6580 }
6581
6582 /* Guest TR. */
6583 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6584 {
6585 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6586 AssertRCReturn(rc, rc);
6587
6588 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6589 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6590 {
6591 rc = VMXLOCAL_READ_SEG(TR, tr);
6592 AssertRCReturn(rc, rc);
6593 }
6594 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6595 }
6596 return rc;
6597}
6598
6599#undef VMXLOCAL_READ_SEG
6600
6601
6602/**
6603 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6604 * context.
6605 *
6606 * @returns VBox status code.
6607 * @param pVCpu Pointer to the VMCPU.
6608 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6609 * out-of-sync. Make sure to update the required fields
6610 * before using them.
6611 *
6612 * @remarks No-long-jump zone!!!
6613 */
6614static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6615{
6616 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6617 {
6618 if (!pVCpu->hm.s.fUsingHyperDR7)
6619 {
6620 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6621 uint32_t u32Val;
6622 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6623 pMixedCtx->dr[7] = u32Val;
6624 }
6625
6626 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6627 }
6628 return VINF_SUCCESS;
6629}
6630
6631
6632/**
6633 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6634 *
6635 * @returns VBox status code.
6636 * @param pVCpu Pointer to the VMCPU.
6637 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6638 * out-of-sync. Make sure to update the required fields
6639 * before using them.
6640 *
6641 * @remarks No-long-jump zone!!!
6642 */
6643static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6644{
6645 NOREF(pMixedCtx);
6646
6647 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6648 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6649 return VINF_SUCCESS;
6650}
6651
6652
6653/**
6654 * Saves the entire guest state from the currently active VMCS into the
6655 * guest-CPU context. This essentially VMREADs all guest-data.
6656 *
6657 * @returns VBox status code.
6658 * @param pVCpu Pointer to the VMCPU.
6659 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6660 * out-of-sync. Make sure to update the required fields
6661 * before using them.
6662 */
6663static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6664{
6665 Assert(pVCpu);
6666 Assert(pMixedCtx);
6667
6668 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6669 return VINF_SUCCESS;
6670
6671 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6672 again on the ring-3 callback path, there is no real need to. */
6673 if (VMMRZCallRing3IsEnabled(pVCpu))
6674 VMMR0LogFlushDisable(pVCpu);
6675 else
6676 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6677 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6678
6679 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6680 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6681
6682 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6683 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6684
6685 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6686 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6687
6688 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6689 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6690
6691 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6692 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6693
6694 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6695 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6696
6697 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6698 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6699
6700 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6701 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6702
6703 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6704 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6705
6706 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6707 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6708
6709 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6710 ("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6711
6712 if (VMMRZCallRing3IsEnabled(pVCpu))
6713 VMMR0LogFlushEnable(pVCpu);
6714
6715 return rc;
6716}
6717
6718
6719/**
6720 * Check per-VM and per-VCPU force flag actions that require us to go back to
6721 * ring-3 for one reason or another.
6722 *
6723 * @returns VBox status code (information status code included).
6724 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6725 * ring-3.
6726 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6727 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6728 * interrupts)
6729 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6730 * all EMTs to be in ring-3.
6731 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6732 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6733 * to the EM loop.
6734 *
6735 * @param pVM Pointer to the VM.
6736 * @param pVCpu Pointer to the VMCPU.
6737 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6738 * out-of-sync. Make sure to update the required fields
6739 * before using them.
6740 */
6741static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6742{
6743 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6744
6745 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6746 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6747 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6748 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6749 {
6750 /* We need the control registers now, make sure the guest-CPU context is updated. */
6751 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6752 AssertRCReturn(rc3, rc3);
6753
6754 /* Pending HM CR3 sync. */
6755 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6756 {
6757 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6758 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6759 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6760 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6761 }
6762
6763 /* Pending HM PAE PDPEs. */
6764 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6765 {
6766 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6767 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6768 }
6769
6770 /* Pending PGM C3 sync. */
6771 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6772 {
6773 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6774 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6775 if (rc2 != VINF_SUCCESS)
6776 {
6777 AssertRC(rc2);
6778 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6779 return rc2;
6780 }
6781 }
6782
6783 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6784 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6785 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6786 {
6787 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6788 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6789 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6790 return rc2;
6791 }
6792
6793 /* Pending VM request packets, such as hardware interrupts. */
6794 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6795 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6796 {
6797 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6798 return VINF_EM_PENDING_REQUEST;
6799 }
6800
6801 /* Pending PGM pool flushes. */
6802 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6803 {
6804 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6805 return VINF_PGM_POOL_FLUSH_PENDING;
6806 }
6807
6808 /* Pending DMA requests. */
6809 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6810 {
6811 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6812 return VINF_EM_RAW_TO_R3;
6813 }
6814 }
6815
6816 return VINF_SUCCESS;
6817}
6818
6819
6820/**
6821 * Converts any TRPM trap into a pending HM event. This is typically used when
6822 * entering from ring-3 (not longjmp returns).
6823 *
6824 * @param pVCpu Pointer to the VMCPU.
6825 */
6826static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6827{
6828 Assert(TRPMHasTrap(pVCpu));
6829 Assert(!pVCpu->hm.s.Event.fPending);
6830
6831 uint8_t uVector;
6832 TRPMEVENT enmTrpmEvent;
6833 RTGCUINT uErrCode;
6834 RTGCUINTPTR GCPtrFaultAddress;
6835 uint8_t cbInstr;
6836
6837 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6838 AssertRC(rc);
6839
6840 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6841 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6842 if (enmTrpmEvent == TRPM_TRAP)
6843 {
6844 switch (uVector)
6845 {
6846 case X86_XCPT_NMI:
6847 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6848 break;
6849
6850 case X86_XCPT_BP:
6851 case X86_XCPT_OF:
6852 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6853 break;
6854
6855 case X86_XCPT_PF:
6856 case X86_XCPT_DF:
6857 case X86_XCPT_TS:
6858 case X86_XCPT_NP:
6859 case X86_XCPT_SS:
6860 case X86_XCPT_GP:
6861 case X86_XCPT_AC:
6862 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6863 /* no break! */
6864 default:
6865 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6866 break;
6867 }
6868 }
6869 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6870 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6871 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6872 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6873 else
6874 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6875
6876 rc = TRPMResetTrap(pVCpu);
6877 AssertRC(rc);
6878 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6879 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6880
6881 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6882 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6883}
6884
6885
6886/**
6887 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6888 * VT-x to execute any instruction.
6889 *
6890 * @param pvCpu Pointer to the VMCPU.
6891 */
6892static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6893{
6894 Assert(pVCpu->hm.s.Event.fPending);
6895
6896 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6897 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6898 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6899 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6900
6901 /* If a trap was already pending, we did something wrong! */
6902 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6903
6904 TRPMEVENT enmTrapType;
6905 switch (uVectorType)
6906 {
6907 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6908 enmTrapType = TRPM_HARDWARE_INT;
6909 break;
6910
6911 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6912 enmTrapType = TRPM_SOFTWARE_INT;
6913 break;
6914
6915 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6916 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6917 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6918 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6919 enmTrapType = TRPM_TRAP;
6920 break;
6921
6922 default:
6923 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6924 enmTrapType = TRPM_32BIT_HACK;
6925 break;
6926 }
6927
6928 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6929
6930 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6931 AssertRC(rc);
6932
6933 if (fErrorCodeValid)
6934 TRPMSetErrorCode(pVCpu, uErrorCode);
6935
6936 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6937 && uVector == X86_XCPT_PF)
6938 {
6939 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6940 }
6941 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6942 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6943 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6944 {
6945 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6946 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6947 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6948 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6949 }
6950 pVCpu->hm.s.Event.fPending = false;
6951}
6952
6953
6954/**
6955 * Does the necessary state syncing before returning to ring-3 for any reason
6956 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6957 *
6958 * @returns VBox status code.
6959 * @param pVM Pointer to the VM.
6960 * @param pVCpu Pointer to the VMCPU.
6961 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6962 * be out-of-sync. Make sure to update the required
6963 * fields before using them.
6964 * @param fSaveGuestState Whether to save the guest state or not.
6965 *
6966 * @remarks No-long-jmp zone!!!
6967 */
6968static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6969{
6970 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6971 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6972
6973 RTCPUID idCpu = RTMpCpuId();
6974 Log4Func(("HostCpuId=%u\n", idCpu));
6975
6976 /*
6977 * !!! IMPORTANT !!!
6978 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6979 */
6980
6981 /* Save the guest state if necessary. */
6982 if ( fSaveGuestState
6983 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6984 {
6985 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6986 AssertRCReturn(rc, rc);
6987 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6988 }
6989
6990 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6991 if (CPUMIsGuestFPUStateActive(pVCpu))
6992 {
6993 /* We shouldn't reload CR0 without saving it first. */
6994 if (!fSaveGuestState)
6995 {
6996 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6997 AssertRCReturn(rc, rc);
6998 }
6999 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
7000 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
7001 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7002 }
7003
7004 /* Restore host debug registers if necessary and resync on next R0 reentry. */
7005#ifdef VBOX_STRICT
7006 if (CPUMIsHyperDebugStateActive(pVCpu))
7007 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7008#endif
7009 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7010 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7011 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7012 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7013
7014#if HC_ARCH_BITS == 64
7015 /* Restore host-state bits that VT-x only restores partially. */
7016 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7017 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7018 {
7019 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7020 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7021 }
7022 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7023#endif
7024
7025#if HC_ARCH_BITS == 64
7026 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7027 if ( pVM->hm.s.fAllow64BitGuests
7028 && pVCpu->hm.s.vmx.fLazyMsrs)
7029 {
7030 /* We shouldn't reload the guest MSRs without saving it first. */
7031 if (!fSaveGuestState)
7032 {
7033 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7034 AssertRCReturn(rc, rc);
7035 }
7036 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7037 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7038 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7039 }
7040#endif
7041
7042 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7043 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7044
7045 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7046 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7047 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7048 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7049 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7050 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7051 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7052 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7053
7054 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7055
7056 /** @todo This partially defeats the purpose of having preemption hooks.
7057 * The problem is, deregistering the hooks should be moved to a place that
7058 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7059 * context.
7060 */
7061 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7062 {
7063 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7064 AssertRCReturn(rc, rc);
7065
7066 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7067 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7068 }
7069 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7070 NOREF(idCpu);
7071
7072 return VINF_SUCCESS;
7073}
7074
7075
7076/**
7077 * Leaves the VT-x session.
7078 *
7079 * @returns VBox status code.
7080 * @param pVM Pointer to the VM.
7081 * @param pVCpu Pointer to the VMCPU.
7082 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7083 * out-of-sync. Make sure to update the required fields
7084 * before using them.
7085 *
7086 * @remarks No-long-jmp zone!!!
7087 */
7088DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7089{
7090 HM_DISABLE_PREEMPT_IF_NEEDED();
7091 HMVMX_ASSERT_CPU_SAFE();
7092 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7093 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7094
7095 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7096 and done this from the VMXR0ThreadCtxCallback(). */
7097 if (!pVCpu->hm.s.fLeaveDone)
7098 {
7099 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7100 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
7101 pVCpu->hm.s.fLeaveDone = true;
7102 }
7103 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7104
7105 /*
7106 * !!! IMPORTANT !!!
7107 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7108 */
7109
7110 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7111 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
7112 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7113 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7114 VMMR0ThreadCtxHooksDeregister(pVCpu);
7115
7116 /* Leave HM context. This takes care of local init (term). */
7117 int rc = HMR0LeaveCpu(pVCpu);
7118
7119 HM_RESTORE_PREEMPT_IF_NEEDED();
7120
7121 return rc;
7122}
7123
7124
7125/**
7126 * Does the necessary state syncing before doing a longjmp to ring-3.
7127 *
7128 * @returns VBox status code.
7129 * @param pVM Pointer to the VM.
7130 * @param pVCpu Pointer to the VMCPU.
7131 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7132 * out-of-sync. Make sure to update the required fields
7133 * before using them.
7134 *
7135 * @remarks No-long-jmp zone!!!
7136 */
7137DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7138{
7139 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7140}
7141
7142
7143/**
7144 * Take necessary actions before going back to ring-3.
7145 *
7146 * An action requires us to go back to ring-3. This function does the necessary
7147 * steps before we can safely return to ring-3. This is not the same as longjmps
7148 * to ring-3, this is voluntary and prepares the guest so it may continue
7149 * executing outside HM (recompiler/IEM).
7150 *
7151 * @returns VBox status code.
7152 * @param pVM Pointer to the VM.
7153 * @param pVCpu Pointer to the VMCPU.
7154 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7155 * out-of-sync. Make sure to update the required fields
7156 * before using them.
7157 * @param rcExit The reason for exiting to ring-3. Can be
7158 * VINF_VMM_UNKNOWN_RING3_CALL.
7159 */
7160static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
7161{
7162 Assert(pVM);
7163 Assert(pVCpu);
7164 Assert(pMixedCtx);
7165 HMVMX_ASSERT_PREEMPT_SAFE();
7166
7167 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7168 {
7169 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7170 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7171 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7172 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7173 }
7174
7175 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7176 VMMRZCallRing3Disable(pVCpu);
7177 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
7178
7179 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7180 if (pVCpu->hm.s.Event.fPending)
7181 {
7182 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7183 Assert(!pVCpu->hm.s.Event.fPending);
7184 }
7185
7186 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7187 and if we're injecting an event we should have a TRPM trap pending. */
7188 Assert(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu));
7189 Assert(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu));
7190
7191 /* Save guest state and restore host state bits. */
7192 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7193 AssertRCReturn(rc, rc);
7194 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7195 /* Thread-context hooks are unregistered at this point!!! */
7196
7197 /* Sync recompiler state. */
7198 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7199 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7200 | CPUM_CHANGED_LDTR
7201 | CPUM_CHANGED_GDTR
7202 | CPUM_CHANGED_IDTR
7203 | CPUM_CHANGED_TR
7204 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7205 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7206 if ( pVM->hm.s.fNestedPaging
7207 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7208 {
7209 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7210 }
7211
7212 Assert(!pVCpu->hm.s.fClearTrapFlag);
7213
7214 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7215 if (rcExit != VINF_EM_RAW_INTERRUPT)
7216 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7217
7218 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7219
7220 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7221 VMMRZCallRing3RemoveNotification(pVCpu);
7222 VMMRZCallRing3Enable(pVCpu);
7223
7224 return rc;
7225}
7226
7227
7228/**
7229 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7230 * longjump to ring-3 and possibly get preempted.
7231 *
7232 * @returns VBox status code.
7233 * @param pVCpu Pointer to the VMCPU.
7234 * @param enmOperation The operation causing the ring-3 longjump.
7235 * @param pvUser Opaque pointer to the guest-CPU context. The data
7236 * may be out-of-sync. Make sure to update the required
7237 * fields before using them.
7238 */
7239DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7240{
7241 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7242 {
7243 /*
7244 * !!! IMPORTANT !!!
7245 * If you modify code here, make sure to check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs
7246 * to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
7247 */
7248 VMMRZCallRing3RemoveNotification(pVCpu);
7249 VMMRZCallRing3Disable(pVCpu);
7250 HM_DISABLE_PREEMPT_IF_NEEDED();
7251
7252 PVM pVM = pVCpu->CTX_SUFF(pVM);
7253 if (CPUMIsGuestFPUStateActive(pVCpu))
7254 CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
7255
7256 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7257
7258#if HC_ARCH_BITS == 64
7259 /* Restore host-state bits that VT-x only restores partially. */
7260 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7261 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7262 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7263 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7264
7265 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7266 if ( pVM->hm.s.fAllow64BitGuests
7267 && pVCpu->hm.s.vmx.fLazyMsrs)
7268 {
7269 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7270 }
7271#endif
7272 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7273 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7274 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7275 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7276 {
7277 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7278 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7279 }
7280
7281 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7282 VMMR0ThreadCtxHooksDeregister(pVCpu);
7283
7284 HMR0LeaveCpu(pVCpu);
7285 HM_RESTORE_PREEMPT_IF_NEEDED();
7286 return VINF_SUCCESS;
7287 }
7288
7289 Assert(pVCpu);
7290 Assert(pvUser);
7291 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7292 HMVMX_ASSERT_PREEMPT_SAFE();
7293
7294 VMMRZCallRing3Disable(pVCpu);
7295 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7296
7297 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7298 enmOperation));
7299
7300 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7301 AssertRCReturn(rc, rc);
7302
7303 VMMRZCallRing3Enable(pVCpu);
7304 return VINF_SUCCESS;
7305}
7306
7307
7308/**
7309 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7310 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7311 *
7312 * @param pVCpu Pointer to the VMCPU.
7313 */
7314DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7315{
7316 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7317 {
7318 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7319 {
7320 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7321 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7322 AssertRC(rc);
7323 Log4(("Setup interrupt-window exiting\n"));
7324 }
7325 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7326}
7327
7328
7329/**
7330 * Clears the interrupt-window exiting control in the VMCS.
7331 *
7332 * @param pVCpu Pointer to the VMCPU.
7333 */
7334DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7335{
7336 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7337 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7338 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7339 AssertRC(rc);
7340 Log4(("Cleared interrupt-window exiting\n"));
7341}
7342
7343
7344/**
7345 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7346 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7347 *
7348 * @param pVCpu Pointer to the VMCPU.
7349 */
7350DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7351{
7352 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7353 {
7354 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7355 {
7356 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7357 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7358 AssertRC(rc);
7359 Log4(("Setup NMI-window exiting\n"));
7360 }
7361 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7362}
7363
7364
7365/**
7366 * Clears the NMI-window exiting control in the VMCS.
7367 *
7368 * @param pVCpu Pointer to the VMCPU.
7369 */
7370DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7371{
7372 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7373 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7374 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7375 AssertRC(rc);
7376 Log4(("Cleared NMI-window exiting\n"));
7377}
7378
7379
7380/**
7381 * Evaluates the event to be delivered to the guest and sets it as the pending
7382 * event.
7383 *
7384 * @param pVCpu Pointer to the VMCPU.
7385 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7386 * out-of-sync. Make sure to update the required fields
7387 * before using them.
7388 */
7389static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7390{
7391 Assert(!pVCpu->hm.s.Event.fPending);
7392
7393 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7394 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7395 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7396 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7397 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7398
7399 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7400 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7401 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7402 Assert(!TRPMHasTrap(pVCpu));
7403
7404 /*
7405 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7406 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7407 */
7408 /** @todo SMI. SMIs take priority over NMIs. */
7409 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7410 {
7411 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7412 if ( !fBlockNmi
7413 && !fBlockSti
7414 && !fBlockMovSS)
7415 {
7416 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7417 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7418 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7419
7420 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7421 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7422 }
7423 else
7424 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7425 }
7426 /*
7427 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
7428 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt.
7429 */
7430 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7431 && !pVCpu->hm.s.fSingleInstruction)
7432 {
7433 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7434 AssertRC(rc);
7435 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7436 if ( !fBlockInt
7437 && !fBlockSti
7438 && !fBlockMovSS)
7439 {
7440 uint8_t u8Interrupt;
7441 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7442 if (RT_SUCCESS(rc))
7443 {
7444 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7445 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7446 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7447
7448 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7449 }
7450 else
7451 {
7452 /** @todo Does this actually happen? If not turn it into an assertion. */
7453 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
7454 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7455 }
7456 }
7457 else
7458 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7459 }
7460}
7461
7462
7463/**
7464 * Sets a pending-debug exception to be delivered to the guest if the guest is
7465 * single-stepping.
7466 *
7467 * @param pVCpu Pointer to the VMCPU.
7468 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7469 * out-of-sync. Make sure to update the required fields
7470 * before using them.
7471 */
7472DECLINLINE(void) hmR0VmxSetPendingDebugXcpt(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7473{
7474 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7475 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7476 {
7477 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7478 AssertRC(rc);
7479 }
7480}
7481
7482
7483/**
7484 * Injects any pending events into the guest if the guest is in a state to
7485 * receive them.
7486 *
7487 * @returns VBox status code (informational status codes included).
7488 * @param pVCpu Pointer to the VMCPU.
7489 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7490 * out-of-sync. Make sure to update the required fields
7491 * before using them.
7492 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7493 * return VINF_EM_DBG_STEPPED if the event was
7494 * dispatched directly.
7495 */
7496static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
7497{
7498 HMVMX_ASSERT_PREEMPT_SAFE();
7499 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7500
7501 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7502 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7503 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7504 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7505
7506 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7507 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7508 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7509 Assert(!TRPMHasTrap(pVCpu));
7510
7511 int rc = VINF_SUCCESS;
7512 if (pVCpu->hm.s.Event.fPending)
7513 {
7514 /*
7515 * Clear any interrupt-window exiting control if we're going to inject an interrupt. Saves one extra
7516 * VM-exit in situations where we previously setup interrupt-window exiting but got other VM-exits and
7517 * ended up enabling interrupts outside VT-x.
7518 */
7519 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7520 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7521 && uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7522 {
7523 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7524 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7525 }
7526
7527#ifdef VBOX_STRICT
7528 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7529 {
7530 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7531 Assert(!fBlockInt);
7532 Assert(!fBlockSti);
7533 Assert(!fBlockMovSS);
7534 }
7535 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7536 {
7537 bool fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7538 Assert(!fBlockSti);
7539 Assert(!fBlockMovSS);
7540 Assert(!fBlockNmi);
7541 }
7542#endif
7543 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7544 (uint8_t)uIntType));
7545 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7546 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping, &uIntrState);
7547 AssertRCReturn(rc, rc);
7548
7549 /* Update the interruptibility-state as it could have been changed by
7550 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7551 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7552 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7553
7554#ifdef VBOX_WITH_STATISTICS
7555 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7556 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7557 else
7558 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7559#endif
7560 }
7561
7562 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7563 if ( fBlockSti
7564 || fBlockMovSS)
7565 {
7566 if ( !pVCpu->hm.s.fSingleInstruction
7567 && !DBGFIsStepping(pVCpu))
7568 {
7569 /*
7570 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7571 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7572 * See Intel spec. 27.3.4 "Saving Non-Register State".
7573 */
7574 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7575 AssertRCReturn(rc2, rc2);
7576 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
7577 }
7578 else if (pMixedCtx->eflags.Bits.u1TF)
7579 {
7580 /*
7581 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7582 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7583 */
7584 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7585 uIntrState = 0;
7586 }
7587 }
7588
7589 /*
7590 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7591 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7592 */
7593 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7594 AssertRC(rc2);
7595
7596 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
7597 NOREF(fBlockMovSS); NOREF(fBlockSti);
7598 return rc;
7599}
7600
7601
7602/**
7603 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7604 *
7605 * @param pVCpu Pointer to the VMCPU.
7606 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7607 * out-of-sync. Make sure to update the required fields
7608 * before using them.
7609 */
7610DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7611{
7612 NOREF(pMixedCtx);
7613 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7614 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7615}
7616
7617
7618/**
7619 * Injects a double-fault (#DF) exception into the VM.
7620 *
7621 * @returns VBox status code (informational status code included).
7622 * @param pVCpu Pointer to the VMCPU.
7623 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7624 * out-of-sync. Make sure to update the required fields
7625 * before using them.
7626 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7627 * and should return VINF_EM_DBG_STEPPED if the event
7628 * is injected directly (register modified by us, not
7629 * by hardware on VM-entry).
7630 * @param puIntrState Pointer to the current guest interruptibility-state.
7631 * This interruptibility-state will be updated if
7632 * necessary. This cannot not be NULL.
7633 */
7634DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7635{
7636 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7637 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7638 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7639 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7640 fStepping, puIntrState);
7641}
7642
7643
7644/**
7645 * Sets a debug (#DB) exception as pending-for-injection into the VM.
7646 *
7647 * @param pVCpu Pointer to the VMCPU.
7648 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7649 * out-of-sync. Make sure to update the required fields
7650 * before using them.
7651 */
7652DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7653{
7654 NOREF(pMixedCtx);
7655 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7656 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7657 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7658}
7659
7660
7661/**
7662 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
7663 *
7664 * @param pVCpu Pointer to the VMCPU.
7665 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7666 * out-of-sync. Make sure to update the required fields
7667 * before using them.
7668 * @param cbInstr The value of RIP that is to be pushed on the guest
7669 * stack.
7670 */
7671DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7672{
7673 NOREF(pMixedCtx);
7674 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7675 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7676 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7677}
7678
7679
7680/**
7681 * Injects a general-protection (#GP) fault into the VM.
7682 *
7683 * @returns VBox status code (informational status code included).
7684 * @param pVCpu Pointer to the VMCPU.
7685 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7686 * out-of-sync. Make sure to update the required fields
7687 * before using them.
7688 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7689 * mode, i.e. in real-mode it's not valid).
7690 * @param u32ErrorCode The error code associated with the #GP.
7691 * @param fStepping Whether we're running in
7692 * hmR0VmxRunGuestCodeStep() and should return
7693 * VINF_EM_DBG_STEPPED if the event is injected
7694 * directly (register modified by us, not by
7695 * hardware on VM-entry).
7696 * @param puIntrState Pointer to the current guest interruptibility-state.
7697 * This interruptibility-state will be updated if
7698 * necessary. This cannot not be NULL.
7699 */
7700DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7701 bool fStepping, uint32_t *puIntrState)
7702{
7703 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7704 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7705 if (fErrorCodeValid)
7706 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7707 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7708 fStepping, puIntrState);
7709}
7710
7711
7712/**
7713 * Sets a general-protection (#GP) exception as pending-for-injection into the
7714 * VM.
7715 *
7716 * @param pVCpu Pointer to the VMCPU.
7717 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7718 * out-of-sync. Make sure to update the required fields
7719 * before using them.
7720 * @param u32ErrorCode The error code associated with the #GP.
7721 */
7722DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7723{
7724 NOREF(pMixedCtx);
7725 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7726 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7727 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7728 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7729}
7730
7731
7732/**
7733 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7734 *
7735 * @param pVCpu Pointer to the VMCPU.
7736 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7737 * out-of-sync. Make sure to update the required fields
7738 * before using them.
7739 * @param uVector The software interrupt vector number.
7740 * @param cbInstr The value of RIP that is to be pushed on the guest
7741 * stack.
7742 */
7743DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7744{
7745 NOREF(pMixedCtx);
7746 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7747 if ( uVector == X86_XCPT_BP
7748 || uVector == X86_XCPT_OF)
7749 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7750 else
7751 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7752 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7753}
7754
7755
7756/**
7757 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7758 * stack.
7759 *
7760 * @returns VBox status code (information status code included).
7761 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7762 * @param pVM Pointer to the VM.
7763 * @param pMixedCtx Pointer to the guest-CPU context.
7764 * @param uValue The value to push to the guest stack.
7765 */
7766DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7767{
7768 /*
7769 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7770 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7771 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7772 */
7773 if (pMixedCtx->sp == 1)
7774 return VINF_EM_RESET;
7775 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7776 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7777 AssertRCReturn(rc, rc);
7778 return rc;
7779}
7780
7781
7782/**
7783 * Injects an event into the guest upon VM-entry by updating the relevant fields
7784 * in the VM-entry area in the VMCS.
7785 *
7786 * @returns VBox status code (informational error codes included).
7787 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7788 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7789 *
7790 * @param pVCpu Pointer to the VMCPU.
7791 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7792 * be out-of-sync. Make sure to update the required
7793 * fields before using them.
7794 * @param u64IntInfo The VM-entry interruption-information field.
7795 * @param cbInstr The VM-entry instruction length in bytes (for
7796 * software interrupts, exceptions and privileged
7797 * software exceptions).
7798 * @param u32ErrCode The VM-entry exception error code.
7799 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7800 * @param puIntrState Pointer to the current guest interruptibility-state.
7801 * This interruptibility-state will be updated if
7802 * necessary. This cannot not be NULL.
7803 * @param fStepping Whether we're running in
7804 * hmR0VmxRunGuestCodeStep() and should return
7805 * VINF_EM_DBG_STEPPED if the event is injected
7806 * directly (register modified by us, not by
7807 * hardware on VM-entry).
7808 *
7809 * @remarks Requires CR0!
7810 * @remarks No-long-jump zone!!!
7811 */
7812static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7813 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *puIntrState)
7814{
7815 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7816 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7817 Assert(puIntrState);
7818 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7819
7820 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7821 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7822
7823#ifdef VBOX_STRICT
7824 /* Validate the error-code-valid bit for hardware exceptions. */
7825 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7826 {
7827 switch (uVector)
7828 {
7829 case X86_XCPT_PF:
7830 case X86_XCPT_DF:
7831 case X86_XCPT_TS:
7832 case X86_XCPT_NP:
7833 case X86_XCPT_SS:
7834 case X86_XCPT_GP:
7835 case X86_XCPT_AC:
7836 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7837 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7838 /* fallthru */
7839 default:
7840 break;
7841 }
7842 }
7843#endif
7844
7845 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7846 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7847 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7848
7849 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7850
7851 /* We require CR0 to check if the guest is in real-mode. */
7852 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7853 AssertRCReturn(rc, rc);
7854
7855 /*
7856 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7857 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7858 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7859 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7860 */
7861 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7862 {
7863 PVM pVM = pVCpu->CTX_SUFF(pVM);
7864 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7865 {
7866 Assert(PDMVmmDevHeapIsEnabled(pVM));
7867 Assert(pVM->hm.s.vmx.pRealModeTSS);
7868
7869 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7870 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7871 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7872 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7873 AssertRCReturn(rc, rc);
7874 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7875
7876 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7877 size_t const cbIdtEntry = sizeof(X86IDTR16);
7878 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7879 {
7880 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7881 if (uVector == X86_XCPT_DF)
7882 return VINF_EM_RESET;
7883
7884 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7885 if (uVector == X86_XCPT_GP)
7886 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
7887
7888 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7889 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7890 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
7891 fStepping, puIntrState);
7892 }
7893
7894 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7895 uint16_t uGuestIp = pMixedCtx->ip;
7896 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7897 {
7898 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7899 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7900 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7901 }
7902 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7903 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7904
7905 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7906 X86IDTR16 IdtEntry;
7907 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7908 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7909 AssertRCReturn(rc, rc);
7910
7911 /* Construct the stack frame for the interrupt/exception handler. */
7912 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7913 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7914 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7915 AssertRCReturn(rc, rc);
7916
7917 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7918 if (rc == VINF_SUCCESS)
7919 {
7920 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7921 pMixedCtx->rip = IdtEntry.offSel;
7922 pMixedCtx->cs.Sel = IdtEntry.uSel;
7923 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7924 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7925 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7926 && uVector == X86_XCPT_PF)
7927 pMixedCtx->cr2 = GCPtrFaultAddress;
7928
7929 /* If any other guest-state bits are changed here, make sure to update
7930 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7931 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7932 | HM_CHANGED_GUEST_RIP
7933 | HM_CHANGED_GUEST_RFLAGS
7934 | HM_CHANGED_GUEST_RSP);
7935
7936 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7937 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7938 {
7939 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7940 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7941 Log4(("Clearing inhibition due to STI.\n"));
7942 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7943 }
7944 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7945 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
7946
7947 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7948 it, if we are returning to ring-3 before executing guest code. */
7949 pVCpu->hm.s.Event.fPending = false;
7950
7951 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
7952 if (fStepping)
7953 rc = VINF_EM_DBG_STEPPED;
7954 }
7955 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
7956 return rc;
7957 }
7958
7959 /*
7960 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7961 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7962 */
7963 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7964 }
7965
7966 /* Validate. */
7967 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7968 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7969 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7970
7971 /* Inject. */
7972 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7973 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7974 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7975 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7976
7977 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7978 && uVector == X86_XCPT_PF)
7979 pMixedCtx->cr2 = GCPtrFaultAddress;
7980
7981 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7982 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7983
7984 AssertRCReturn(rc, rc);
7985 return rc;
7986}
7987
7988
7989/**
7990 * Clears the interrupt-window exiting control in the VMCS and if necessary
7991 * clears the current event in the VMCS as well.
7992 *
7993 * @returns VBox status code.
7994 * @param pVCpu Pointer to the VMCPU.
7995 *
7996 * @remarks Use this function only to clear events that have not yet been
7997 * delivered to the guest but are injected in the VMCS!
7998 * @remarks No-long-jump zone!!!
7999 */
8000static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
8001{
8002 int rc;
8003 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8004
8005 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8006 {
8007 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8008 Assert(!pVCpu->hm.s.Event.fPending);
8009 }
8010
8011 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8012 {
8013 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8014 Assert(!pVCpu->hm.s.Event.fPending);
8015 }
8016
8017 if (!pVCpu->hm.s.Event.fPending)
8018 return;
8019
8020#ifdef VBOX_STRICT
8021 uint32_t u32EntryInfo;
8022 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8023 AssertRC(rc);
8024 Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
8025#endif
8026
8027 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8028 AssertRC(rc);
8029
8030 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
8031 AssertRC(rc);
8032
8033 /* We deliberately don't clear "hm.s.Event.fPending" here, it's taken
8034 care of in hmR0VmxExitToRing3() converting the pending event to TRPM. */
8035}
8036
8037
8038/**
8039 * Enters the VT-x session.
8040 *
8041 * @returns VBox status code.
8042 * @param pVM Pointer to the VM.
8043 * @param pVCpu Pointer to the VMCPU.
8044 * @param pCpu Pointer to the CPU info struct.
8045 */
8046VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8047{
8048 AssertPtr(pVM);
8049 AssertPtr(pVCpu);
8050 Assert(pVM->hm.s.vmx.fSupported);
8051 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8052 NOREF(pCpu); NOREF(pVM);
8053
8054 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8055 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8056
8057#ifdef VBOX_STRICT
8058 /* Make sure we're in VMX root mode. */
8059 RTCCUINTREG u32HostCR4 = ASMGetCR4();
8060 if (!(u32HostCR4 & X86_CR4_VMXE))
8061 {
8062 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8063 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8064 }
8065#endif
8066
8067 /*
8068 * Load the VCPU's VMCS as the current (and active) one.
8069 */
8070 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8071 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8072 if (RT_FAILURE(rc))
8073 return rc;
8074
8075 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8076 pVCpu->hm.s.fLeaveDone = false;
8077 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8078
8079 return VINF_SUCCESS;
8080}
8081
8082
8083/**
8084 * The thread-context callback (only on platforms which support it).
8085 *
8086 * @param enmEvent The thread-context event.
8087 * @param pVCpu Pointer to the VMCPU.
8088 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8089 * @thread EMT(pVCpu)
8090 */
8091VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8092{
8093 NOREF(fGlobalInit);
8094
8095 switch (enmEvent)
8096 {
8097 case RTTHREADCTXEVENT_PREEMPTING:
8098 {
8099 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8100 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
8101 VMCPU_ASSERT_EMT(pVCpu);
8102
8103 PVM pVM = pVCpu->CTX_SUFF(pVM);
8104 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8105
8106 /* No longjmps (logger flushes, locks) in this fragile context. */
8107 VMMRZCallRing3Disable(pVCpu);
8108 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8109
8110 /*
8111 * Restore host-state (FPU, debug etc.)
8112 */
8113 if (!pVCpu->hm.s.fLeaveDone)
8114 {
8115 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8116 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8117 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
8118 pVCpu->hm.s.fLeaveDone = true;
8119 }
8120
8121 /* Leave HM context, takes care of local init (term). */
8122 int rc = HMR0LeaveCpu(pVCpu);
8123 AssertRC(rc); NOREF(rc);
8124
8125 /* Restore longjmp state. */
8126 VMMRZCallRing3Enable(pVCpu);
8127 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
8128 break;
8129 }
8130
8131 case RTTHREADCTXEVENT_RESUMED:
8132 {
8133 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8134 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
8135 VMCPU_ASSERT_EMT(pVCpu);
8136
8137 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8138 VMMRZCallRing3Disable(pVCpu);
8139 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8140
8141 /* Initialize the bare minimum state required for HM. This takes care of
8142 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8143 int rc = HMR0EnterCpu(pVCpu);
8144 AssertRC(rc);
8145 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8146
8147 /* Load the active VMCS as the current one. */
8148 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8149 {
8150 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8151 AssertRC(rc); NOREF(rc);
8152 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8153 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8154 }
8155 pVCpu->hm.s.fLeaveDone = false;
8156
8157 /* Restore longjmp state. */
8158 VMMRZCallRing3Enable(pVCpu);
8159 break;
8160 }
8161
8162 default:
8163 break;
8164 }
8165}
8166
8167
8168/**
8169 * Saves the host state in the VMCS host-state.
8170 * Sets up the VM-exit MSR-load area.
8171 *
8172 * The CPU state will be loaded from these fields on every successful VM-exit.
8173 *
8174 * @returns VBox status code.
8175 * @param pVM Pointer to the VM.
8176 * @param pVCpu Pointer to the VMCPU.
8177 *
8178 * @remarks No-long-jump zone!!!
8179 */
8180static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8181{
8182 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8183
8184 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8185 return VINF_SUCCESS;
8186
8187 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8188 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8189
8190 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8191 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8192
8193 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8194 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8195
8196 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8197 return rc;
8198}
8199
8200
8201/**
8202 * Saves the host state in the VMCS host-state.
8203 *
8204 * @returns VBox status code.
8205 * @param pVM Pointer to the VM.
8206 * @param pVCpu Pointer to the VMCPU.
8207 *
8208 * @remarks No-long-jump zone!!!
8209 */
8210VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8211{
8212 AssertPtr(pVM);
8213 AssertPtr(pVCpu);
8214
8215 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8216
8217 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8218 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8219 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8220 return hmR0VmxSaveHostState(pVM, pVCpu);
8221}
8222
8223
8224/**
8225 * Loads the guest state into the VMCS guest-state area.
8226 *
8227 * The will typically be done before VM-entry when the guest-CPU state and the
8228 * VMCS state may potentially be out of sync.
8229 *
8230 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8231 * VM-entry controls.
8232 * Sets up the appropriate VMX non-root function to execute guest code based on
8233 * the guest CPU mode.
8234 *
8235 * @returns VBox status code.
8236 * @param pVM Pointer to the VM.
8237 * @param pVCpu Pointer to the VMCPU.
8238 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8239 * out-of-sync. Make sure to update the required fields
8240 * before using them.
8241 *
8242 * @remarks No-long-jump zone!!!
8243 */
8244static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8245{
8246 AssertPtr(pVM);
8247 AssertPtr(pVCpu);
8248 AssertPtr(pMixedCtx);
8249 HMVMX_ASSERT_PREEMPT_SAFE();
8250
8251 VMMRZCallRing3Disable(pVCpu);
8252 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8253
8254 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8255
8256 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8257
8258 /* Determine real-on-v86 mode. */
8259 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8260 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8261 && CPUMIsGuestInRealModeEx(pMixedCtx))
8262 {
8263 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8264 }
8265
8266 /*
8267 * Load the guest-state into the VMCS.
8268 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8269 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8270 */
8271 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8272 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8273
8274 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8275 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8276 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8277
8278 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8279 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8280 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8281
8282 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8283 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8284
8285 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8286 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8287
8288 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8289 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8290 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8291
8292 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8293 determine we don't have to swap EFER after all. */
8294 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8295 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8296
8297 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8298 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8299
8300 /*
8301 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8302 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8303 */
8304 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8305 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8306
8307 /* Clear any unused and reserved bits. */
8308 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8309
8310 VMMRZCallRing3Enable(pVCpu);
8311
8312 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8313 return rc;
8314}
8315
8316
8317/**
8318 * Loads the state shared between the host and guest into the VMCS.
8319 *
8320 * @param pVM Pointer to the VM.
8321 * @param pVCpu Pointer to the VMCPU.
8322 * @param pCtx Pointer to the guest-CPU context.
8323 *
8324 * @remarks No-long-jump zone!!!
8325 */
8326static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8327{
8328 NOREF(pVM);
8329
8330 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8331 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8332
8333 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8334 {
8335 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8336 AssertRC(rc);
8337 }
8338
8339 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8340 {
8341 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8342 AssertRC(rc);
8343
8344 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8345 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8346 {
8347 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8348 AssertRC(rc);
8349 }
8350 }
8351
8352 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8353 {
8354#if HC_ARCH_BITS == 64
8355 if (pVM->hm.s.fAllow64BitGuests)
8356 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8357#endif
8358 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8359 }
8360
8361 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8362 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8363}
8364
8365
8366/**
8367 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8368 *
8369 * @param pVM Pointer to the VM.
8370 * @param pVCpu Pointer to the VMCPU.
8371 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8372 * out-of-sync. Make sure to update the required fields
8373 * before using them.
8374 */
8375DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8376{
8377 HMVMX_ASSERT_PREEMPT_SAFE();
8378
8379 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8380#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8381 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8382#endif
8383
8384 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8385 {
8386 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8387 AssertRC(rc);
8388 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8389 }
8390 else if (HMCPU_CF_VALUE(pVCpu))
8391 {
8392 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8393 AssertRC(rc);
8394 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8395 }
8396
8397 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8398 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8399 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8400 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8401}
8402
8403
8404/**
8405 * Does the preparations before executing guest code in VT-x.
8406 *
8407 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8408 * recompiler/IEM. We must be cautious what we do here regarding committing
8409 * guest-state information into the VMCS assuming we assuredly execute the
8410 * guest in VT-x mode.
8411 *
8412 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8413 * the common-state (TRPM/forceflags), we must undo those changes so that the
8414 * recompiler/IEM can (and should) use them when it resumes guest execution.
8415 * Otherwise such operations must be done when we can no longer exit to ring-3.
8416 *
8417 * @returns Strict VBox status code.
8418 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8419 * have been disabled.
8420 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8421 * double-fault into the guest.
8422 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8423 * dispatched directly.
8424 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8425 *
8426 * @param pVM Pointer to the VM.
8427 * @param pVCpu Pointer to the VMCPU.
8428 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8429 * out-of-sync. Make sure to update the required fields
8430 * before using them.
8431 * @param pVmxTransient Pointer to the VMX transient structure.
8432 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8433 * us ignore some of the reasons for returning to
8434 * ring-3, and return VINF_EM_DBG_STEPPED if event
8435 * dispatching took place.
8436 */
8437static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8438{
8439 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8440
8441#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8442 PGMRZDynMapFlushAutoSet(pVCpu);
8443#endif
8444
8445 /* Check force flag actions that might require us to go back to ring-3. */
8446 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
8447 if (rc != VINF_SUCCESS)
8448 return rc;
8449
8450#ifndef IEM_VERIFICATION_MODE_FULL
8451 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8452 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8453 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8454 {
8455 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8456 RTGCPHYS GCPhysApicBase;
8457 GCPhysApicBase = pMixedCtx->msrApicBase;
8458 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8459
8460 /* Unalias any existing mapping. */
8461 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8462 AssertRCReturn(rc, rc);
8463
8464 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8465 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
8466 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8467 AssertRCReturn(rc, rc);
8468
8469 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8470 }
8471#endif /* !IEM_VERIFICATION_MODE_FULL */
8472
8473 if (TRPMHasTrap(pVCpu))
8474 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8475 else if (!pVCpu->hm.s.Event.fPending)
8476 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8477
8478 /*
8479 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8480 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8481 */
8482 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, fStepping);
8483 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8484 {
8485 Assert(rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
8486 return rc;
8487 }
8488
8489 /*
8490 * Load the guest state bits, we can handle longjmps/getting preempted here.
8491 *
8492 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8493 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8494 * Hence, this needs to be done -after- injection of events.
8495 */
8496 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8497
8498 /*
8499 * No longjmps to ring-3 from this point on!!!
8500 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8501 * This also disables flushing of the R0-logger instance (if any).
8502 */
8503 VMMRZCallRing3Disable(pVCpu);
8504
8505 /*
8506 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8507 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8508 *
8509 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8510 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8511 *
8512 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8513 * executing guest code.
8514 */
8515 pVmxTransient->uEflags = ASMIntDisableFlags();
8516 if ( ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8517 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8518 && ( !fStepping /* Optimized for the non-stepping case, of course. */
8519 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8520 {
8521 hmR0VmxClearEventVmcs(pVCpu);
8522 ASMSetFlags(pVmxTransient->uEflags);
8523 VMMRZCallRing3Enable(pVCpu);
8524 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8525 return VINF_EM_RAW_TO_R3;
8526 }
8527
8528 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8529 {
8530 hmR0VmxClearEventVmcs(pVCpu);
8531 ASMSetFlags(pVmxTransient->uEflags);
8532 VMMRZCallRing3Enable(pVCpu);
8533 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8534 return VINF_EM_RAW_INTERRUPT;
8535 }
8536
8537 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8538 pVCpu->hm.s.Event.fPending = false;
8539
8540 return VINF_SUCCESS;
8541}
8542
8543
8544/**
8545 * Prepares to run guest code in VT-x and we've committed to doing so. This
8546 * means there is no backing out to ring-3 or anywhere else at this
8547 * point.
8548 *
8549 * @param pVM Pointer to the VM.
8550 * @param pVCpu Pointer to the VMCPU.
8551 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8552 * out-of-sync. Make sure to update the required fields
8553 * before using them.
8554 * @param pVmxTransient Pointer to the VMX transient structure.
8555 *
8556 * @remarks Called with preemption disabled.
8557 * @remarks No-long-jump zone!!!
8558 */
8559static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8560{
8561 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8562 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8563 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8564
8565 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8566 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8567
8568#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8569 if (!CPUMIsGuestFPUStateActive(pVCpu))
8570 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8571 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8572#endif
8573
8574 if ( pVCpu->hm.s.fUseGuestFpu
8575 && !CPUMIsGuestFPUStateActive(pVCpu))
8576 {
8577 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8578 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8579 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8580 }
8581
8582 /*
8583 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8584 */
8585 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8586 && pVCpu->hm.s.vmx.cMsrs > 0)
8587 {
8588 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8589 }
8590
8591 /*
8592 * Load the host state bits as we may've been preempted (only happens when
8593 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8594 */
8595 /** @todo Why should hmR0VmxSetupVMRunHandler() changing pfnStartVM have
8596 * any effect to the host state needing to be saved? */
8597 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8598 {
8599 /* This ASSUMES that pfnStartVM has been set up already. */
8600 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8601 AssertRC(rc);
8602 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8603 }
8604 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8605
8606 /*
8607 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8608 */
8609 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8610 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8611 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8612
8613 /* Store status of the shared guest-host state at the time of VM-entry. */
8614#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8615 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8616 {
8617 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8618 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8619 }
8620 else
8621#endif
8622 {
8623 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8624 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8625 }
8626 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8627
8628 /*
8629 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8630 */
8631 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8632 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8633
8634 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8635 RTCPUID idCurrentCpu = pCpu->idCpu;
8636 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8637 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8638 {
8639 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
8640 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8641 }
8642
8643 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8644 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8645 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8646 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8647
8648 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8649
8650 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8651 to start executing. */
8652
8653 /*
8654 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8655 */
8656 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8657 {
8658 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8659 {
8660 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8661 AssertRC(rc2);
8662 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8663 bool fMsrUpdated = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu),
8664 true /* fUpdateHostMsr */);
8665 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8666 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8667 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8668 }
8669 else
8670 {
8671 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8672 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8673 }
8674 }
8675
8676#ifdef VBOX_STRICT
8677 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8678 hmR0VmxCheckHostEferMsr(pVCpu);
8679 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8680#endif
8681#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8682 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8683 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8684 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8685#endif
8686}
8687
8688
8689/**
8690 * Performs some essential restoration of state after running guest code in
8691 * VT-x.
8692 *
8693 * @param pVM Pointer to the VM.
8694 * @param pVCpu Pointer to the VMCPU.
8695 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8696 * out-of-sync. Make sure to update the required fields
8697 * before using them.
8698 * @param pVmxTransient Pointer to the VMX transient structure.
8699 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8700 *
8701 * @remarks Called with interrupts disabled, and returns with interrups enabled!
8702 *
8703 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8704 * unconditionally when it is safe to do so.
8705 */
8706static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8707{
8708 NOREF(pVM);
8709
8710 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8711
8712 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8713 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8714 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8715 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8716 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8717 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8718
8719 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8720 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
8721
8722 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8723 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8724 Assert(!(ASMGetFlags() & X86_EFL_IF));
8725 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8726
8727#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8728 if (CPUMIsGuestFPUStateActive(pVCpu))
8729 {
8730 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8731 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8732 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8733 }
8734#endif
8735
8736#if HC_ARCH_BITS == 64
8737 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8738#endif
8739 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8740#ifdef VBOX_STRICT
8741 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8742#endif
8743 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8744 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8745
8746 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8747 uint32_t uExitReason;
8748 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8749 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8750 AssertRC(rc);
8751 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8752 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8753
8754 /* Update the VM-exit history array. */
8755 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8756
8757 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8758 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8759 {
8760 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8761 pVmxTransient->fVMEntryFailed));
8762 return;
8763 }
8764
8765 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8766 {
8767 /** @todo We can optimize this by only syncing with our force-flags when
8768 * really needed and keeping the VMCS state as it is for most
8769 * VM-exits. */
8770 /* Update the guest interruptibility-state from the VMCS. */
8771 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8772
8773#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8774 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8775 AssertRC(rc);
8776#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8777 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8778 AssertRC(rc);
8779#endif
8780
8781 /*
8782 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8783 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8784 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8785 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8786 */
8787 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8788 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8789 {
8790 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8791 AssertRC(rc);
8792 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8793 }
8794 }
8795}
8796
8797
8798/**
8799 * Runs the guest code using VT-x the normal way.
8800 *
8801 * @returns VBox status code.
8802 * @param pVM Pointer to the VM.
8803 * @param pVCpu Pointer to the VMCPU.
8804 * @param pCtx Pointer to the guest-CPU context.
8805 *
8806 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8807 */
8808static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8809{
8810 VMXTRANSIENT VmxTransient;
8811 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8812 int rc = VERR_INTERNAL_ERROR_5;
8813 uint32_t cLoops = 0;
8814
8815 for (;; cLoops++)
8816 {
8817 Assert(!HMR0SuspendPending());
8818 HMVMX_ASSERT_CPU_SAFE();
8819
8820 /* Preparatory work for running guest code, this may force us to return
8821 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8822 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8823 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8824 if (rc != VINF_SUCCESS)
8825 break;
8826
8827 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8828 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8829 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8830
8831 /* Restore any residual host-state and save any bits shared between host
8832 and guest into the guest-CPU state. Re-enables interrupts! */
8833 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8834
8835 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8836 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8837 {
8838 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8839 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8840 return rc;
8841 }
8842
8843 /* Profile the VM-exit. */
8844 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8845 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8846 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8847 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8848 HMVMX_START_EXIT_DISPATCH_PROF();
8849
8850 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8851 if (RT_UNLIKELY(VBOXVMM_R0_HMVMX_VMEXIT_ENABLED()))
8852 {
8853 hmR0VmxReadExitQualificationVmcs(pVCpu, &VmxTransient);
8854 hmR0VmxSaveGuestState(pVCpu, pCtx);
8855 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, VmxTransient.uExitReason, VmxTransient.uExitQualification);
8856 }
8857
8858 /* Handle the VM-exit. */
8859#ifdef HMVMX_USE_FUNCTION_TABLE
8860 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8861#else
8862 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8863#endif
8864 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8865 if (rc != VINF_SUCCESS)
8866 break;
8867 if (cLoops > pVM->hm.s.cMaxResumeLoops)
8868 {
8869 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8870 rc = VINF_EM_RAW_INTERRUPT;
8871 break;
8872 }
8873 }
8874
8875 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8876 return rc;
8877}
8878
8879
8880/**
8881 * Single steps guest code using VT-x.
8882 *
8883 * @returns VBox status code.
8884 * @param pVM Pointer to the VM.
8885 * @param pVCpu Pointer to the VMCPU.
8886 * @param pCtx Pointer to the guest-CPU context.
8887 *
8888 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8889 */
8890static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8891{
8892 VMXTRANSIENT VmxTransient;
8893 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8894 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8895 uint32_t cLoops = 0;
8896 uint16_t uCsStart = pCtx->cs.Sel;
8897 uint64_t uRipStart = pCtx->rip;
8898
8899 for (;; cLoops++)
8900 {
8901 Assert(!HMR0SuspendPending());
8902 HMVMX_ASSERT_CPU_SAFE();
8903
8904 /* Preparatory work for running guest code, this may force us to return
8905 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8906 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8907 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, true /* fStepping */);
8908 if (rcStrict != VINF_SUCCESS)
8909 break;
8910
8911 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8912 rcStrict = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8913 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8914
8915 /* Restore any residual host-state and save any bits shared between host
8916 and guest into the guest-CPU state. Re-enables interrupts! */
8917 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
8918
8919 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8920 if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
8921 {
8922 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8923 hmR0VmxReportWorldSwitchError(pVM, pVCpu, VBOXSTRICTRC_TODO(rcStrict), pCtx, &VmxTransient);
8924 return VBOXSTRICTRC_TODO(rcStrict);
8925 }
8926
8927 /* Profile the VM-exit. */
8928 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8929 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8930 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8931 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8932 HMVMX_START_EXIT_DISPATCH_PROF();
8933
8934 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8935 if (RT_UNLIKELY(VBOXVMM_R0_HMVMX_VMEXIT_ENABLED()))
8936 {
8937 hmR0VmxReadExitQualificationVmcs(pVCpu, &VmxTransient);
8938 hmR0VmxSaveGuestState(pVCpu, pCtx);
8939 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, VmxTransient.uExitReason, VmxTransient.uExitQualification);
8940 }
8941
8942 /* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitStep(). */
8943 rcStrict = hmR0VmxHandleExitStep(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, uCsStart, uRipStart);
8944 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8945 if (rcStrict != VINF_SUCCESS)
8946 break;
8947 if (cLoops > pVM->hm.s.cMaxResumeLoops)
8948 {
8949 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8950 rcStrict = VINF_EM_RAW_INTERRUPT;
8951 break;
8952 }
8953
8954 /*
8955 * Did the RIP change, if so, consider it a single step.
8956 * Otherwise, make sure one of the TFs gets set.
8957 */
8958 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8959 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8960 AssertRCReturn(rc2, rc2);
8961 if ( pCtx->rip != uRipStart
8962 || pCtx->cs.Sel != uCsStart)
8963 {
8964 rcStrict = VINF_EM_DBG_STEPPED;
8965 break;
8966 }
8967 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8968 }
8969
8970 /*
8971 * Clear the X86_EFL_TF if necessary.
8972 */
8973 if (pVCpu->hm.s.fClearTrapFlag)
8974 {
8975 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8976 AssertRCReturn(rc2, rc2);
8977 pVCpu->hm.s.fClearTrapFlag = false;
8978 pCtx->eflags.Bits.u1TF = 0;
8979 }
8980 /** @todo there seems to be issues with the resume flag when the monitor trap
8981 * flag is pending without being used. Seen early in bios init when
8982 * accessing APIC page in protected mode. */
8983
8984 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8985 return VBOXSTRICTRC_TODO(rcStrict);
8986}
8987
8988
8989/**
8990 * Runs the guest code using VT-x.
8991 *
8992 * @returns VBox status code.
8993 * @param pVM Pointer to the VM.
8994 * @param pVCpu Pointer to the VMCPU.
8995 * @param pCtx Pointer to the guest-CPU context.
8996 */
8997VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8998{
8999 Assert(VMMRZCallRing3IsEnabled(pVCpu));
9000 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
9001 HMVMX_ASSERT_PREEMPT_SAFE();
9002
9003 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
9004
9005 int rc;
9006 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
9007 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
9008 else
9009 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
9010
9011 if (rc == VERR_EM_INTERPRETER)
9012 rc = VINF_EM_RAW_EMULATE_INSTR;
9013 else if (rc == VINF_EM_RESET)
9014 rc = VINF_EM_TRIPLE_FAULT;
9015
9016 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
9017 if (RT_FAILURE(rc2))
9018 {
9019 pVCpu->hm.s.u32HMError = rc;
9020 rc = rc2;
9021 }
9022 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
9023 return rc;
9024}
9025
9026
9027#ifndef HMVMX_USE_FUNCTION_TABLE
9028DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
9029{
9030#ifdef DEBUG_ramshankar
9031# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
9032# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
9033#endif
9034 int rc;
9035 switch (rcReason)
9036 {
9037 case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); */ rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9038 case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); */ rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9039 case VMX_EXIT_IO_INSTR: /* SVVMCS(); */ rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9040 case VMX_EXIT_CPUID: /* SVVMCS(); */ rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9041 case VMX_EXIT_RDTSC: /* SVVMCS(); */ rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9042 case VMX_EXIT_RDTSCP: /* SVVMCS(); */ rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9043 case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9044 case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); */ rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9045 case VMX_EXIT_MOV_CRX: /* SVVMCS(); */ rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9046 case VMX_EXIT_EXT_INT: /* SVVMCS(); */ rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9047 case VMX_EXIT_INT_WINDOW: /* SVVMCS(); */ rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9048 case VMX_EXIT_MWAIT: /* SVVMCS(); */ rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9049 case VMX_EXIT_MONITOR: /* SVVMCS(); */ rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9050 case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); */ rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9051 case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); */ rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9052 case VMX_EXIT_RDMSR: /* SVVMCS(); */ rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9053 case VMX_EXIT_WRMSR: /* SVVMCS(); */ rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9054 case VMX_EXIT_MOV_DRX: /* SVVMCS(); */ rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9055 case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); */ rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9056 case VMX_EXIT_HLT: /* SVVMCS(); */ rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9057 case VMX_EXIT_INVD: /* SVVMCS(); */ rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9058 case VMX_EXIT_INVLPG: /* SVVMCS(); */ rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9059 case VMX_EXIT_RSM: /* SVVMCS(); */ rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9060 case VMX_EXIT_MTF: /* SVVMCS(); */ rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9061 case VMX_EXIT_PAUSE: /* SVVMCS(); */ rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9062 case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9063 case VMX_EXIT_TR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9064 case VMX_EXIT_WBINVD: /* SVVMCS(); */ rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9065 case VMX_EXIT_XSETBV: /* SVVMCS(); */ rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9066 case VMX_EXIT_RDRAND: /* SVVMCS(); */ rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9067 case VMX_EXIT_INVPCID: /* SVVMCS(); */ rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9068 case VMX_EXIT_GETSEC: /* SVVMCS(); */ rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9069 case VMX_EXIT_RDPMC: /* SVVMCS(); */ rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9070 case VMX_EXIT_VMCALL: /* SVVMCS(); */ rc = hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9071
9072 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
9073 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
9074 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
9075 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
9076 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
9077 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
9078 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
9079 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
9080 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
9081
9082 case VMX_EXIT_VMCLEAR:
9083 case VMX_EXIT_VMLAUNCH:
9084 case VMX_EXIT_VMPTRLD:
9085 case VMX_EXIT_VMPTRST:
9086 case VMX_EXIT_VMREAD:
9087 case VMX_EXIT_VMRESUME:
9088 case VMX_EXIT_VMWRITE:
9089 case VMX_EXIT_VMXOFF:
9090 case VMX_EXIT_VMXON:
9091 case VMX_EXIT_INVEPT:
9092 case VMX_EXIT_INVVPID:
9093 case VMX_EXIT_VMFUNC:
9094 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
9095 break;
9096 default:
9097 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
9098 break;
9099 }
9100 return rc;
9101}
9102#endif /* !HMVMX_USE_FUNCTION_TABLE */
9103
9104
9105/**
9106 * Single-stepping VM-exit filtering.
9107 *
9108 * This is preprocessing the exits and deciding whether we've gotten far enough
9109 * to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit handling is
9110 * performed.
9111 *
9112 * @returns Strict VBox status code.
9113 * @param pVCpu The virtual CPU of the calling EMT.
9114 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9115 * out-of-sync. Make sure to update the required
9116 * fields before using them.
9117 * @param pVmxTransient Pointer to the VMX-transient structure.
9118 * @param uExitReason The VM-exit reason.
9119 */
9120DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitStep(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9121 uint32_t uExitReason, uint16_t uCsStart, uint64_t uRipStart)
9122{
9123 switch (uExitReason)
9124 {
9125 case VMX_EXIT_XCPT_OR_NMI:
9126 {
9127 /* Check for host NMI. */
9128 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9129 AssertRCReturn(rc2, rc2);
9130 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9131 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9132 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9133 /* fall thru */
9134 }
9135
9136 case VMX_EXIT_EPT_MISCONFIG:
9137 case VMX_EXIT_TRIPLE_FAULT:
9138 case VMX_EXIT_APIC_ACCESS:
9139 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9140 case VMX_EXIT_TASK_SWITCH:
9141
9142 /* Instruction specific VM-exits: */
9143 case VMX_EXIT_IO_INSTR:
9144 case VMX_EXIT_CPUID:
9145 case VMX_EXIT_RDTSC:
9146 case VMX_EXIT_RDTSCP:
9147 case VMX_EXIT_MOV_CRX:
9148 case VMX_EXIT_MWAIT:
9149 case VMX_EXIT_MONITOR:
9150 case VMX_EXIT_RDMSR:
9151 case VMX_EXIT_WRMSR:
9152 case VMX_EXIT_MOV_DRX:
9153 case VMX_EXIT_HLT:
9154 case VMX_EXIT_INVD:
9155 case VMX_EXIT_INVLPG:
9156 case VMX_EXIT_RSM:
9157 case VMX_EXIT_PAUSE:
9158 case VMX_EXIT_XDTR_ACCESS:
9159 case VMX_EXIT_TR_ACCESS:
9160 case VMX_EXIT_WBINVD:
9161 case VMX_EXIT_XSETBV:
9162 case VMX_EXIT_RDRAND:
9163 case VMX_EXIT_INVPCID:
9164 case VMX_EXIT_GETSEC:
9165 case VMX_EXIT_RDPMC:
9166 case VMX_EXIT_VMCALL:
9167 case VMX_EXIT_VMCLEAR:
9168 case VMX_EXIT_VMLAUNCH:
9169 case VMX_EXIT_VMPTRLD:
9170 case VMX_EXIT_VMPTRST:
9171 case VMX_EXIT_VMREAD:
9172 case VMX_EXIT_VMRESUME:
9173 case VMX_EXIT_VMWRITE:
9174 case VMX_EXIT_VMXOFF:
9175 case VMX_EXIT_VMXON:
9176 case VMX_EXIT_INVEPT:
9177 case VMX_EXIT_INVVPID:
9178 case VMX_EXIT_VMFUNC:
9179 {
9180 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9181 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9182 AssertRCReturn(rc2, rc2);
9183 if ( pMixedCtx->rip != uRipStart
9184 || pMixedCtx->cs.Sel != uCsStart)
9185 return VINF_EM_DBG_STEPPED;
9186 break;
9187 }
9188 }
9189
9190 /*
9191 * Normal processing.
9192 */
9193#ifdef HMVMX_USE_FUNCTION_TABLE
9194 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9195#else
9196 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9197#endif
9198}
9199
9200
9201#ifdef DEBUG
9202/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
9203# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
9204 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
9205
9206# define HMVMX_ASSERT_PREEMPT_CPUID() \
9207 do \
9208 { \
9209 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
9210 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
9211 } while (0)
9212
9213# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9214 do { \
9215 AssertPtr(pVCpu); \
9216 AssertPtr(pMixedCtx); \
9217 AssertPtr(pVmxTransient); \
9218 Assert(pVmxTransient->fVMEntryFailed == false); \
9219 Assert(ASMIntAreEnabled()); \
9220 HMVMX_ASSERT_PREEMPT_SAFE(); \
9221 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
9222 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
9223 HMVMX_ASSERT_PREEMPT_SAFE(); \
9224 if (VMMR0IsLogFlushDisabled(pVCpu)) \
9225 HMVMX_ASSERT_PREEMPT_CPUID(); \
9226 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9227 } while (0)
9228
9229# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
9230 do { \
9231 Log4Func(("\n")); \
9232 } while (0)
9233#else /* Release builds */
9234# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9235 do { \
9236 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9237 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
9238 } while (0)
9239# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
9240#endif
9241
9242
9243/**
9244 * Advances the guest RIP after reading it from the VMCS.
9245 *
9246 * @returns VBox status code.
9247 * @param pVCpu Pointer to the VMCPU.
9248 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9249 * out-of-sync. Make sure to update the required fields
9250 * before using them.
9251 * @param pVmxTransient Pointer to the VMX transient structure.
9252 *
9253 * @remarks No-long-jump zone!!!
9254 */
9255DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9256{
9257 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9258 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9259 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9260 AssertRCReturn(rc, rc);
9261
9262 pMixedCtx->rip += pVmxTransient->cbInstr;
9263 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9264
9265 /*
9266 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
9267 * pending debug exception field as it takes care of priority of events.
9268 *
9269 * See Intel spec. 32.2.1 "Debug Exceptions".
9270 */
9271 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
9272
9273 return rc;
9274}
9275
9276
9277/**
9278 * Tries to determine what part of the guest-state VT-x has deemed as invalid
9279 * and update error record fields accordingly.
9280 *
9281 * @return VMX_IGS_* return codes.
9282 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9283 * wrong with the guest state.
9284 *
9285 * @param pVM Pointer to the VM.
9286 * @param pVCpu Pointer to the VMCPU.
9287 * @param pCtx Pointer to the guest-CPU state.
9288 *
9289 * @remarks This function assumes our cache of the VMCS controls
9290 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9291 */
9292static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9293{
9294#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9295#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9296 uError = (err); \
9297 break; \
9298 } else do { } while (0)
9299
9300 int rc;
9301 uint32_t uError = VMX_IGS_ERROR;
9302 uint32_t u32Val;
9303 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9304
9305 do
9306 {
9307 /*
9308 * CR0.
9309 */
9310 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9311 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9312 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
9313 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9314 if (fUnrestrictedGuest)
9315 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
9316
9317 uint32_t u32GuestCR0;
9318 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
9319 AssertRCBreak(rc);
9320 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
9321 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
9322 if ( !fUnrestrictedGuest
9323 && (u32GuestCR0 & X86_CR0_PG)
9324 && !(u32GuestCR0 & X86_CR0_PE))
9325 {
9326 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9327 }
9328
9329 /*
9330 * CR4.
9331 */
9332 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9333 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9334
9335 uint32_t u32GuestCR4;
9336 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
9337 AssertRCBreak(rc);
9338 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
9339 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
9340
9341 /*
9342 * IA32_DEBUGCTL MSR.
9343 */
9344 uint64_t u64Val;
9345 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9346 AssertRCBreak(rc);
9347 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9348 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9349 {
9350 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9351 }
9352 uint64_t u64DebugCtlMsr = u64Val;
9353
9354#ifdef VBOX_STRICT
9355 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9356 AssertRCBreak(rc);
9357 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
9358#endif
9359 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
9360
9361 /*
9362 * RIP and RFLAGS.
9363 */
9364 uint32_t u32Eflags;
9365#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9366 if (HMVMX_IS_64BIT_HOST_MODE())
9367 {
9368 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9369 AssertRCBreak(rc);
9370 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9371 if ( !fLongModeGuest
9372 || !pCtx->cs.Attr.n.u1Long)
9373 {
9374 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9375 }
9376 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9377 * must be identical if the "IA-32e mode guest" VM-entry
9378 * control is 1 and CS.L is 1. No check applies if the
9379 * CPU supports 64 linear-address bits. */
9380
9381 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9382 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9383 AssertRCBreak(rc);
9384 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9385 VMX_IGS_RFLAGS_RESERVED);
9386 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9387 u32Eflags = u64Val;
9388 }
9389 else
9390#endif
9391 {
9392 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9393 AssertRCBreak(rc);
9394 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9395 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9396 }
9397
9398 if ( fLongModeGuest
9399 || ( fUnrestrictedGuest
9400 && !(u32GuestCR0 & X86_CR0_PE)))
9401 {
9402 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9403 }
9404
9405 uint32_t u32EntryInfo;
9406 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9407 AssertRCBreak(rc);
9408 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9409 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9410 {
9411 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9412 }
9413
9414 /*
9415 * 64-bit checks.
9416 */
9417#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9418 if (HMVMX_IS_64BIT_HOST_MODE())
9419 {
9420 if ( fLongModeGuest
9421 && !fUnrestrictedGuest)
9422 {
9423 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9424 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9425 }
9426
9427 if ( !fLongModeGuest
9428 && (u32GuestCR4 & X86_CR4_PCIDE))
9429 {
9430 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9431 }
9432
9433 /** @todo CR3 field must be such that bits 63:52 and bits in the range
9434 * 51:32 beyond the processor's physical-address width are 0. */
9435
9436 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9437 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9438 {
9439 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9440 }
9441
9442 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9443 AssertRCBreak(rc);
9444 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9445
9446 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9447 AssertRCBreak(rc);
9448 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9449 }
9450#endif
9451
9452 /*
9453 * PERF_GLOBAL MSR.
9454 */
9455 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
9456 {
9457 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9458 AssertRCBreak(rc);
9459 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9460 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9461 }
9462
9463 /*
9464 * PAT MSR.
9465 */
9466 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
9467 {
9468 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9469 AssertRCBreak(rc);
9470 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9471 for (unsigned i = 0; i < 8; i++)
9472 {
9473 uint8_t u8Val = (u64Val & 0xff);
9474 if ( u8Val != 0 /* UC */
9475 && u8Val != 1 /* WC */
9476 && u8Val != 4 /* WT */
9477 && u8Val != 5 /* WP */
9478 && u8Val != 6 /* WB */
9479 && u8Val != 7 /* UC- */)
9480 {
9481 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9482 }
9483 u64Val >>= 8;
9484 }
9485 }
9486
9487 /*
9488 * EFER MSR.
9489 */
9490 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
9491 {
9492 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9493 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9494 AssertRCBreak(rc);
9495 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9496 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9497 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
9498 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9499 HMVMX_CHECK_BREAK( fUnrestrictedGuest
9500 || !(u32GuestCR0 & X86_CR0_PG)
9501 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9502 VMX_IGS_EFER_LMA_LME_MISMATCH);
9503 }
9504
9505 /*
9506 * Segment registers.
9507 */
9508 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9509 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9510 if (!(u32Eflags & X86_EFL_VM))
9511 {
9512 /* CS */
9513 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9514 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9515 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9516 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9517 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9518 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9519 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9520 /* CS cannot be loaded with NULL in protected mode. */
9521 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9522 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9523 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
9524 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9525 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
9526 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9527 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9528 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9529 else
9530 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9531
9532 /* SS */
9533 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9534 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9535 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9536 if ( !(pCtx->cr0 & X86_CR0_PE)
9537 || pCtx->cs.Attr.n.u4Type == 3)
9538 {
9539 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9540 }
9541 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9542 {
9543 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9544 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9545 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9546 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9547 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9548 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9549 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9550 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9551 }
9552
9553 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
9554 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9555 {
9556 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9557 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9558 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9559 || pCtx->ds.Attr.n.u4Type > 11
9560 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9561 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9562 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9563 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9564 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9565 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9566 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9567 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9568 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9569 }
9570 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9571 {
9572 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9573 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9574 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9575 || pCtx->es.Attr.n.u4Type > 11
9576 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9577 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9578 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9579 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9580 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9581 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9582 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9583 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9584 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9585 }
9586 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9587 {
9588 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9589 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9590 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9591 || pCtx->fs.Attr.n.u4Type > 11
9592 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9593 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9594 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9595 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9596 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9597 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9598 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9599 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9600 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9601 }
9602 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9603 {
9604 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9605 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9606 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9607 || pCtx->gs.Attr.n.u4Type > 11
9608 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9609 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9610 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9611 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9612 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9613 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9614 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9615 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9616 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9617 }
9618 /* 64-bit capable CPUs. */
9619#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9620 if (HMVMX_IS_64BIT_HOST_MODE())
9621 {
9622 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9623 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9624 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9625 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9626 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9627 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9628 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9629 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9630 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9631 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9632 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9633 }
9634#endif
9635 }
9636 else
9637 {
9638 /* V86 mode checks. */
9639 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9640 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9641 {
9642 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9643 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9644 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9645 }
9646 else
9647 {
9648 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9649 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9650 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9651 }
9652
9653 /* CS */
9654 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9655 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9656 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9657 /* SS */
9658 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9659 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9660 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9661 /* DS */
9662 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9663 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9664 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9665 /* ES */
9666 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9667 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9668 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9669 /* FS */
9670 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9671 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9672 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9673 /* GS */
9674 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9675 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9676 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9677 /* 64-bit capable CPUs. */
9678#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9679 if (HMVMX_IS_64BIT_HOST_MODE())
9680 {
9681 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9682 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9683 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9684 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9685 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9686 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9687 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9688 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9689 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9690 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9691 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9692 }
9693#endif
9694 }
9695
9696 /*
9697 * TR.
9698 */
9699 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9700 /* 64-bit capable CPUs. */
9701#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9702 if (HMVMX_IS_64BIT_HOST_MODE())
9703 {
9704 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9705 }
9706#endif
9707 if (fLongModeGuest)
9708 {
9709 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9710 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9711 }
9712 else
9713 {
9714 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9715 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9716 VMX_IGS_TR_ATTR_TYPE_INVALID);
9717 }
9718 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9719 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9720 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9721 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9722 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9723 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9724 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9725 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9726
9727 /*
9728 * GDTR and IDTR.
9729 */
9730#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9731 if (HMVMX_IS_64BIT_HOST_MODE())
9732 {
9733 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9734 AssertRCBreak(rc);
9735 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9736
9737 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9738 AssertRCBreak(rc);
9739 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9740 }
9741#endif
9742
9743 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9744 AssertRCBreak(rc);
9745 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9746
9747 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9748 AssertRCBreak(rc);
9749 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9750
9751 /*
9752 * Guest Non-Register State.
9753 */
9754 /* Activity State. */
9755 uint32_t u32ActivityState;
9756 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9757 AssertRCBreak(rc);
9758 HMVMX_CHECK_BREAK( !u32ActivityState
9759 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9760 VMX_IGS_ACTIVITY_STATE_INVALID);
9761 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9762 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9763 uint32_t u32IntrState;
9764 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9765 AssertRCBreak(rc);
9766 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9767 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9768 {
9769 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9770 }
9771
9772 /** @todo Activity state and injecting interrupts. Left as a todo since we
9773 * currently don't use activity states but ACTIVE. */
9774
9775 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9776 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9777
9778 /* Guest interruptibility-state. */
9779 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9780 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9781 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9782 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9783 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9784 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9785 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9786 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9787 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9788 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9789 {
9790 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9791 {
9792 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9793 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9794 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9795 }
9796 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9797 {
9798 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9799 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9800 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9801 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9802 }
9803 }
9804 /** @todo Assumes the processor is not in SMM. */
9805 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9806 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9807 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9808 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9809 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9810 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9811 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9812 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9813 {
9814 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9815 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9816 }
9817
9818 /* Pending debug exceptions. */
9819 if (HMVMX_IS_64BIT_HOST_MODE())
9820 {
9821 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9822 AssertRCBreak(rc);
9823 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9824 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9825 u32Val = u64Val; /* For pending debug exceptions checks below. */
9826 }
9827 else
9828 {
9829 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9830 AssertRCBreak(rc);
9831 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9832 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9833 }
9834
9835 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9836 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9837 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9838 {
9839 if ( (u32Eflags & X86_EFL_TF)
9840 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9841 {
9842 /* Bit 14 is PendingDebug.BS. */
9843 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9844 }
9845 if ( !(u32Eflags & X86_EFL_TF)
9846 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9847 {
9848 /* Bit 14 is PendingDebug.BS. */
9849 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9850 }
9851 }
9852
9853 /* VMCS link pointer. */
9854 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9855 AssertRCBreak(rc);
9856 if (u64Val != UINT64_C(0xffffffffffffffff))
9857 {
9858 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9859 /** @todo Bits beyond the processor's physical-address width MBZ. */
9860 /** @todo 32-bit located in memory referenced by value of this field (as a
9861 * physical address) must contain the processor's VMCS revision ID. */
9862 /** @todo SMM checks. */
9863 }
9864
9865 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9866 * not using Nested Paging? */
9867 if ( pVM->hm.s.fNestedPaging
9868 && !fLongModeGuest
9869 && CPUMIsGuestInPAEModeEx(pCtx))
9870 {
9871 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9872 AssertRCBreak(rc);
9873 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9874
9875 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9876 AssertRCBreak(rc);
9877 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9878
9879 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9880 AssertRCBreak(rc);
9881 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9882
9883 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9884 AssertRCBreak(rc);
9885 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9886 }
9887
9888 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9889 if (uError == VMX_IGS_ERROR)
9890 uError = VMX_IGS_REASON_NOT_FOUND;
9891 } while (0);
9892
9893 pVCpu->hm.s.u32HMError = uError;
9894 return uError;
9895
9896#undef HMVMX_ERROR_BREAK
9897#undef HMVMX_CHECK_BREAK
9898}
9899
9900/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9901/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9902/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9903
9904/** @name VM-exit handlers.
9905 * @{
9906 */
9907
9908/**
9909 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9910 */
9911HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9912{
9913 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9914 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9915 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9916 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9917 return VINF_SUCCESS;
9918 return VINF_EM_RAW_INTERRUPT;
9919}
9920
9921
9922/**
9923 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9924 */
9925HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9926{
9927 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9928 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9929
9930 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9931 AssertRCReturn(rc, rc);
9932
9933 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9934 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9935 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9936 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9937
9938 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9939 {
9940 /*
9941 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9942 * anything we inject is not going to cause a VM-exit directly for the event being injected.
9943 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9944 *
9945 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9946 */
9947 VMXDispatchHostNmi();
9948 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9949 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9950 return VINF_SUCCESS;
9951 }
9952
9953 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9954 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9955 if (RT_UNLIKELY(rc != VINF_SUCCESS))
9956 {
9957 if (rc == VINF_HM_DOUBLE_FAULT)
9958 rc = VINF_SUCCESS;
9959 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9960 return rc;
9961 }
9962
9963 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9964 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9965 switch (uIntType)
9966 {
9967 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9968 Assert(uVector == X86_XCPT_DB);
9969 /* no break */
9970 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9971 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9972 /* no break */
9973 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9974 {
9975 switch (uVector)
9976 {
9977 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9978 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9979 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9980 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9981 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9982 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9983#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9984 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9985 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9986 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9987 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9988 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9989 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9990 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9991 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9992 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9993 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9994 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
9995 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9996#endif
9997 default:
9998 {
9999 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10000 AssertRCReturn(rc, rc);
10001
10002 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
10003 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10004 {
10005 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
10006 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
10007 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
10008
10009 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10010 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10011 AssertRCReturn(rc, rc);
10012 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
10013 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
10014 0 /* GCPtrFaultAddress */);
10015 AssertRCReturn(rc, rc);
10016 }
10017 else
10018 {
10019 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
10020 pVCpu->hm.s.u32HMError = uVector;
10021 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10022 }
10023 break;
10024 }
10025 }
10026 break;
10027 }
10028
10029 default:
10030 {
10031 pVCpu->hm.s.u32HMError = uExitIntInfo;
10032 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
10033 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
10034 break;
10035 }
10036 }
10037 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10038 return rc;
10039}
10040
10041
10042/**
10043 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
10044 */
10045HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10046{
10047 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10048
10049 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
10050 hmR0VmxClearIntWindowExitVmcs(pVCpu);
10051
10052 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
10053 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
10054 return VINF_SUCCESS;
10055}
10056
10057
10058/**
10059 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
10060 */
10061HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10062{
10063 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10064 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
10065 {
10066 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
10067 HMVMX_RETURN_UNEXPECTED_EXIT();
10068 }
10069
10070 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
10071
10072 /*
10073 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
10074 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
10075 */
10076 uint32_t uIntrState = 0;
10077 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10078 AssertRCReturn(rc, rc);
10079
10080 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
10081 if ( fBlockSti
10082 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
10083 {
10084 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10085 }
10086
10087 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
10088 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
10089
10090 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
10091 return VINF_SUCCESS;
10092}
10093
10094
10095/**
10096 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
10097 */
10098HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10099{
10100 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10101 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
10102 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10103}
10104
10105
10106/**
10107 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
10108 */
10109HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10110{
10111 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10112 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
10113 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10114}
10115
10116
10117/**
10118 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
10119 */
10120HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10121{
10122 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10123 PVM pVM = pVCpu->CTX_SUFF(pVM);
10124 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10125 if (RT_LIKELY(rc == VINF_SUCCESS))
10126 {
10127 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10128 Assert(pVmxTransient->cbInstr == 2);
10129 }
10130 else
10131 {
10132 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
10133 rc = VERR_EM_INTERPRETER;
10134 }
10135 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
10136 return rc;
10137}
10138
10139
10140/**
10141 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
10142 */
10143HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10144{
10145 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10146 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
10147 AssertRCReturn(rc, rc);
10148
10149 if (pMixedCtx->cr4 & X86_CR4_SMXE)
10150 return VINF_EM_RAW_EMULATE_INSTR;
10151
10152 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
10153 HMVMX_RETURN_UNEXPECTED_EXIT();
10154}
10155
10156
10157/**
10158 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
10159 */
10160HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10161{
10162 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10163 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10164 AssertRCReturn(rc, rc);
10165
10166 PVM pVM = pVCpu->CTX_SUFF(pVM);
10167 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10168 if (RT_LIKELY(rc == VINF_SUCCESS))
10169 {
10170 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10171 Assert(pVmxTransient->cbInstr == 2);
10172 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10173 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10174 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10175 }
10176 else
10177 rc = VERR_EM_INTERPRETER;
10178 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10179 return rc;
10180}
10181
10182
10183/**
10184 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
10185 */
10186HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10187{
10188 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10189 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10190 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
10191 AssertRCReturn(rc, rc);
10192
10193 PVM pVM = pVCpu->CTX_SUFF(pVM);
10194 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
10195 if (RT_LIKELY(rc == VINF_SUCCESS))
10196 {
10197 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10198 Assert(pVmxTransient->cbInstr == 3);
10199 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10200 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10201 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10202 }
10203 else
10204 {
10205 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
10206 rc = VERR_EM_INTERPRETER;
10207 }
10208 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10209 return rc;
10210}
10211
10212
10213/**
10214 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
10215 */
10216HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10217{
10218 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10219 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10220 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
10221 AssertRCReturn(rc, rc);
10222
10223 PVM pVM = pVCpu->CTX_SUFF(pVM);
10224 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10225 if (RT_LIKELY(rc == VINF_SUCCESS))
10226 {
10227 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10228 Assert(pVmxTransient->cbInstr == 2);
10229 }
10230 else
10231 {
10232 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
10233 rc = VERR_EM_INTERPRETER;
10234 }
10235 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
10236 return rc;
10237}
10238
10239
10240/**
10241 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
10242 */
10243HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10244{
10245 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10246
10247 int rc = VERR_NOT_SUPPORTED;
10248 if (GIMAreHypercallsEnabled(pVCpu))
10249 {
10250 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10251 AssertRCReturn(rc, rc);
10252
10253 rc = GIMHypercall(pVCpu, pMixedCtx);
10254 }
10255 if (rc != VINF_SUCCESS)
10256 {
10257 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10258 rc = VINF_SUCCESS;
10259 }
10260
10261 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
10262 return rc;
10263}
10264
10265
10266/**
10267 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
10268 */
10269HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10270{
10271 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10272 PVM pVM = pVCpu->CTX_SUFF(pVM);
10273 Assert(!pVM->hm.s.fNestedPaging);
10274
10275 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10276 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10277 AssertRCReturn(rc, rc);
10278
10279 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
10280 rc = VBOXSTRICTRC_VAL(rc2);
10281 if (RT_LIKELY(rc == VINF_SUCCESS))
10282 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10283 else
10284 {
10285 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
10286 pVmxTransient->uExitQualification, rc));
10287 }
10288 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
10289 return rc;
10290}
10291
10292
10293/**
10294 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
10295 */
10296HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10297{
10298 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10299 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10300 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10301 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10302 AssertRCReturn(rc, rc);
10303
10304 PVM pVM = pVCpu->CTX_SUFF(pVM);
10305 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10306 if (RT_LIKELY(rc == VINF_SUCCESS))
10307 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10308 else
10309 {
10310 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
10311 rc = VERR_EM_INTERPRETER;
10312 }
10313 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
10314 return rc;
10315}
10316
10317
10318/**
10319 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
10320 */
10321HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10322{
10323 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10324 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10325 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10326 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10327 AssertRCReturn(rc, rc);
10328
10329 PVM pVM = pVCpu->CTX_SUFF(pVM);
10330 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10331 rc = VBOXSTRICTRC_VAL(rc2);
10332 if (RT_LIKELY( rc == VINF_SUCCESS
10333 || rc == VINF_EM_HALT))
10334 {
10335 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10336 AssertRCReturn(rc3, rc3);
10337
10338 if ( rc == VINF_EM_HALT
10339 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
10340 {
10341 rc = VINF_SUCCESS;
10342 }
10343 }
10344 else
10345 {
10346 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
10347 rc = VERR_EM_INTERPRETER;
10348 }
10349 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
10350 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
10351 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
10352 return rc;
10353}
10354
10355
10356/**
10357 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
10358 */
10359HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10360{
10361 /*
10362 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
10363 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
10364 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
10365 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
10366 */
10367 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10368 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10369 HMVMX_RETURN_UNEXPECTED_EXIT();
10370}
10371
10372
10373/**
10374 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
10375 */
10376HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10377{
10378 /*
10379 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
10380 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
10381 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
10382 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
10383 */
10384 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10385 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10386 HMVMX_RETURN_UNEXPECTED_EXIT();
10387}
10388
10389
10390/**
10391 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
10392 */
10393HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10394{
10395 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
10396 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10397 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10398 HMVMX_RETURN_UNEXPECTED_EXIT();
10399}
10400
10401
10402/**
10403 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
10404 */
10405HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10406{
10407 /*
10408 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
10409 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
10410 * See Intel spec. 25.3 "Other Causes of VM-exits".
10411 */
10412 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10413 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10414 HMVMX_RETURN_UNEXPECTED_EXIT();
10415}
10416
10417
10418/**
10419 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
10420 * VM-exit.
10421 */
10422HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10423{
10424 /*
10425 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
10426 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
10427 *
10428 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
10429 * See Intel spec. "23.8 Restrictions on VMX operation".
10430 */
10431 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10432 return VINF_SUCCESS;
10433}
10434
10435
10436/**
10437 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
10438 * VM-exit.
10439 */
10440HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10441{
10442 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10443 return VINF_EM_RESET;
10444}
10445
10446
10447/**
10448 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
10449 */
10450HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10451{
10452 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10453 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
10454 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10455 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10456 AssertRCReturn(rc, rc);
10457
10458 pMixedCtx->rip++;
10459 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10460 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
10461 rc = VINF_SUCCESS;
10462 else
10463 rc = VINF_EM_HALT;
10464
10465 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10466 if (rc != VINF_SUCCESS)
10467 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
10468 return rc;
10469}
10470
10471
10472/**
10473 * VM-exit handler for instructions that result in a #UD exception delivered to
10474 * the guest.
10475 */
10476HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10477{
10478 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10479 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10480 return VINF_SUCCESS;
10481}
10482
10483
10484/**
10485 * VM-exit handler for expiry of the VMX preemption timer.
10486 */
10487HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10488{
10489 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10490
10491 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
10492 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10493
10494 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
10495 PVM pVM = pVCpu->CTX_SUFF(pVM);
10496 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
10497 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
10498 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
10499}
10500
10501
10502/**
10503 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
10504 */
10505HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10506{
10507 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10508
10509 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
10510 /** @todo check if XSETBV is supported by the recompiler. */
10511 return VERR_EM_INTERPRETER;
10512}
10513
10514
10515/**
10516 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
10517 */
10518HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10519{
10520 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10521
10522 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
10523 /** @todo implement EMInterpretInvpcid() */
10524 return VERR_EM_INTERPRETER;
10525}
10526
10527
10528/**
10529 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
10530 * Error VM-exit.
10531 */
10532HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10533{
10534 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10535 AssertRCReturn(rc, rc);
10536
10537 rc = hmR0VmxCheckVmcsCtls(pVCpu);
10538 AssertRCReturn(rc, rc);
10539
10540 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10541 NOREF(uInvalidReason);
10542
10543#ifdef VBOX_STRICT
10544 uint32_t uIntrState;
10545 HMVMXHCUINTREG uHCReg;
10546 uint64_t u64Val;
10547 uint32_t u32Val;
10548
10549 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
10550 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
10551 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
10552 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10553 AssertRCReturn(rc, rc);
10554
10555 Log4(("uInvalidReason %u\n", uInvalidReason));
10556 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
10557 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
10558 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
10559 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
10560
10561 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
10562 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
10563 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
10564 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
10565 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
10566 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10567 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
10568 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
10569 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
10570 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10571 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
10572 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
10573#else
10574 NOREF(pVmxTransient);
10575#endif
10576
10577 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10578 return VERR_VMX_INVALID_GUEST_STATE;
10579}
10580
10581
10582/**
10583 * VM-exit handler for VM-entry failure due to an MSR-load
10584 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
10585 */
10586HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10587{
10588 NOREF(pVmxTransient);
10589 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10590 HMVMX_RETURN_UNEXPECTED_EXIT();
10591}
10592
10593
10594/**
10595 * VM-exit handler for VM-entry failure due to a machine-check event
10596 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
10597 */
10598HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10599{
10600 NOREF(pVmxTransient);
10601 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10602 HMVMX_RETURN_UNEXPECTED_EXIT();
10603}
10604
10605
10606/**
10607 * VM-exit handler for all undefined reasons. Should never ever happen.. in
10608 * theory.
10609 */
10610HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10611{
10612 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
10613 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
10614 return VERR_VMX_UNDEFINED_EXIT_CODE;
10615}
10616
10617
10618/**
10619 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
10620 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
10621 * Conditional VM-exit.
10622 */
10623HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10624{
10625 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10626
10627 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
10628 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
10629 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
10630 return VERR_EM_INTERPRETER;
10631 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10632 HMVMX_RETURN_UNEXPECTED_EXIT();
10633}
10634
10635
10636/**
10637 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
10638 */
10639HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10640{
10641 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10642
10643 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
10644 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
10645 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
10646 return VERR_EM_INTERPRETER;
10647 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10648 HMVMX_RETURN_UNEXPECTED_EXIT();
10649}
10650
10651
10652/**
10653 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
10654 */
10655HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10656{
10657 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10658
10659 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
10660 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10661 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10662 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10663 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10664 {
10665 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10666 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10667 }
10668 AssertRCReturn(rc, rc);
10669 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10670
10671#ifdef VBOX_STRICT
10672 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
10673 {
10674 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
10675 && pMixedCtx->ecx != MSR_K6_EFER)
10676 {
10677 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10678 HMVMX_RETURN_UNEXPECTED_EXIT();
10679 }
10680# if HC_ARCH_BITS == 64
10681 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
10682 && hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10683 {
10684 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10685 HMVMX_RETURN_UNEXPECTED_EXIT();
10686 }
10687# endif
10688 }
10689#endif
10690
10691 PVM pVM = pVCpu->CTX_SUFF(pVM);
10692 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10693 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
10694 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
10695 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
10696 if (RT_LIKELY(rc == VINF_SUCCESS))
10697 {
10698 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10699 Assert(pVmxTransient->cbInstr == 2);
10700 }
10701 return rc;
10702}
10703
10704
10705/**
10706 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10707 */
10708HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10709{
10710 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10711 PVM pVM = pVCpu->CTX_SUFF(pVM);
10712 int rc = VINF_SUCCESS;
10713
10714 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
10715 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10716 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10717 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10718 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10719 {
10720 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10721 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10722 }
10723 AssertRCReturn(rc, rc);
10724 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
10725
10726 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10727 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10728 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10729
10730 if (RT_LIKELY(rc == VINF_SUCCESS))
10731 {
10732 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10733
10734 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
10735 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10736 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10737 {
10738 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10739 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10740 EMInterpretWrmsr() changes it. */
10741 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10742 }
10743 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10744 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10745 else if (pMixedCtx->ecx == MSR_K6_EFER)
10746 {
10747 /*
10748 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
10749 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
10750 * the other bits as well, SCE and NXE. See @bugref{7368}.
10751 */
10752 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
10753 }
10754
10755 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10756 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10757 {
10758 switch (pMixedCtx->ecx)
10759 {
10760 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10761 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10762 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10763 case MSR_K8_FS_BASE: /* no break */
10764 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10765 case MSR_K6_EFER: /* already handled above */ break;
10766 default:
10767 {
10768 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10769 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10770#if HC_ARCH_BITS == 64
10771 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10772 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10773#endif
10774 break;
10775 }
10776 }
10777 }
10778#ifdef VBOX_STRICT
10779 else
10780 {
10781 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10782 switch (pMixedCtx->ecx)
10783 {
10784 case MSR_IA32_SYSENTER_CS:
10785 case MSR_IA32_SYSENTER_EIP:
10786 case MSR_IA32_SYSENTER_ESP:
10787 case MSR_K8_FS_BASE:
10788 case MSR_K8_GS_BASE:
10789 {
10790 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10791 HMVMX_RETURN_UNEXPECTED_EXIT();
10792 }
10793
10794 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10795 default:
10796 {
10797 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10798 {
10799 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
10800 if (pMixedCtx->ecx != MSR_K6_EFER)
10801 {
10802 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10803 pMixedCtx->ecx));
10804 HMVMX_RETURN_UNEXPECTED_EXIT();
10805 }
10806 }
10807
10808#if HC_ARCH_BITS == 64
10809 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10810 {
10811 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10812 HMVMX_RETURN_UNEXPECTED_EXIT();
10813 }
10814#endif
10815 break;
10816 }
10817 }
10818 }
10819#endif /* VBOX_STRICT */
10820 }
10821 return rc;
10822}
10823
10824
10825/**
10826 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10827 */
10828HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10829{
10830 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10831
10832 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10833 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10834 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10835 return VERR_EM_INTERPRETER;
10836 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10837 HMVMX_RETURN_UNEXPECTED_EXIT();
10838}
10839
10840
10841/**
10842 * VM-exit handler for when the TPR value is lowered below the specified
10843 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10844 */
10845HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10846{
10847 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10848 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10849
10850 /*
10851 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10852 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10853 * resume guest execution.
10854 */
10855 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10856 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10857 return VINF_SUCCESS;
10858}
10859
10860
10861/**
10862 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10863 * VM-exit.
10864 *
10865 * @retval VINF_SUCCESS when guest execution can continue.
10866 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10867 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10868 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10869 * recompiler.
10870 */
10871HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10872{
10873 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10874 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10875 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10876 AssertRCReturn(rc, rc);
10877
10878 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
10879 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10880 PVM pVM = pVCpu->CTX_SUFF(pVM);
10881 switch (uAccessType)
10882 {
10883 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10884 {
10885#if 0
10886 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10887 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10888#else
10889 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10890 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10891 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10892#endif
10893 AssertRCReturn(rc, rc);
10894
10895 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10896 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10897 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10898 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
10899
10900 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10901 {
10902 case 0: /* CR0 */
10903 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10904 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10905 break;
10906 case 2: /* CR2 */
10907 /* Nothing to do here, CR2 it's not part of the VMCS. */
10908 break;
10909 case 3: /* CR3 */
10910 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10911 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10912 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10913 break;
10914 case 4: /* CR4 */
10915 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10916 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10917 break;
10918 case 8: /* CR8 */
10919 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10920 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10921 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10922 break;
10923 default:
10924 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10925 break;
10926 }
10927
10928 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10929 break;
10930 }
10931
10932 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10933 {
10934 /* EMInterpretCRxRead() requires EFER MSR, CS. */
10935 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10936 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10937 AssertRCReturn(rc, rc);
10938 Assert( !pVM->hm.s.fNestedPaging
10939 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10940 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10941
10942 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10943 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10944 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10945
10946 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10947 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10948 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10949 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10950 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10951 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10952 break;
10953 }
10954
10955 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10956 {
10957 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10958 AssertRCReturn(rc, rc);
10959 rc = EMInterpretCLTS(pVM, pVCpu);
10960 AssertRCReturn(rc, rc);
10961 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10962 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10963 Log4(("CRX CLTS write rc=%d\n", rc));
10964 break;
10965 }
10966
10967 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10968 {
10969 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10970 AssertRCReturn(rc, rc);
10971 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10972 if (RT_LIKELY(rc == VINF_SUCCESS))
10973 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10974 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10975 Log4(("CRX LMSW write rc=%d\n", rc));
10976 break;
10977 }
10978
10979 default:
10980 {
10981 AssertMsgFailed(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType));
10982 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10983 }
10984 }
10985
10986 /* Validate possible error codes. */
10987 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
10988 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10989 if (RT_SUCCESS(rc))
10990 {
10991 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10992 AssertRCReturn(rc2, rc2);
10993 }
10994
10995 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10996 return rc;
10997}
10998
10999
11000/**
11001 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
11002 * VM-exit.
11003 */
11004HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11005{
11006 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11007 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
11008
11009 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11010 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11011 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11012 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
11013 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
11014 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
11015 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
11016 AssertRCReturn(rc2, rc2);
11017
11018 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
11019 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
11020 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
11021 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
11022 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
11023 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
11024 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11025 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
11026
11027 /* I/O operation lookup arrays. */
11028 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
11029 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
11030
11031 VBOXSTRICTRC rcStrict;
11032 uint32_t const cbValue = s_aIOSizes[uIOWidth];
11033 uint32_t const cbInstr = pVmxTransient->cbInstr;
11034 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
11035 PVM pVM = pVCpu->CTX_SUFF(pVM);
11036 if (fIOString)
11037 {
11038#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
11039 /*
11040 * INS/OUTS - I/O String instruction.
11041 *
11042 * Use instruction-information if available, otherwise fall back on
11043 * interpreting the instruction.
11044 */
11045 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
11046 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
11047 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
11048 {
11049 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11050 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
11051 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11052 AssertRCReturn(rc2, rc2);
11053 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
11054 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
11055 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
11056 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
11057 if (fIOWrite)
11058 {
11059 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
11060 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
11061 }
11062 else
11063 {
11064 /*
11065 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
11066 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
11067 * See Intel Instruction spec. for "INS".
11068 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
11069 */
11070 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
11071 }
11072 }
11073 else
11074 {
11075 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
11076 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11077 AssertRCReturn(rc2, rc2);
11078 rcStrict = IEMExecOne(pVCpu);
11079 }
11080 /** @todo IEM needs to be setting these flags somehow. */
11081 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11082 fUpdateRipAlready = true;
11083#else
11084 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11085 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
11086 if (RT_SUCCESS(rcStrict))
11087 {
11088 if (fIOWrite)
11089 {
11090 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
11091 (DISCPUMODE)pDis->uAddrMode, cbValue);
11092 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
11093 }
11094 else
11095 {
11096 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
11097 (DISCPUMODE)pDis->uAddrMode, cbValue);
11098 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
11099 }
11100 }
11101 else
11102 {
11103 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
11104 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
11105 }
11106#endif
11107 }
11108 else
11109 {
11110 /*
11111 * IN/OUT - I/O instruction.
11112 */
11113 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
11114 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
11115 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
11116 if (fIOWrite)
11117 {
11118 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
11119 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11120 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11121 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
11122 }
11123 else
11124 {
11125 uint32_t u32Result = 0;
11126 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
11127 if (IOM_SUCCESS(rcStrict))
11128 {
11129 /* Save result of I/O IN instr. in AL/AX/EAX. */
11130 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
11131 }
11132 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11133 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11134 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
11135 }
11136 }
11137
11138 if (IOM_SUCCESS(rcStrict))
11139 {
11140 if (!fUpdateRipAlready)
11141 {
11142 pMixedCtx->rip += cbInstr;
11143 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11144 }
11145
11146 /*
11147 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
11148 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
11149 */
11150 if (fIOString)
11151 {
11152 /** @todo Single-step for INS/OUTS with REP prefix? */
11153 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
11154 }
11155 else if (fStepping)
11156 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11157
11158 /*
11159 * If any I/O breakpoints are armed, we need to check if one triggered
11160 * and take appropriate action.
11161 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
11162 */
11163 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11164 AssertRCReturn(rc2, rc2);
11165
11166 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
11167 * execution engines about whether hyper BPs and such are pending. */
11168 uint32_t const uDr7 = pMixedCtx->dr[7];
11169 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
11170 && X86_DR7_ANY_RW_IO(uDr7)
11171 && (pMixedCtx->cr4 & X86_CR4_DE))
11172 || DBGFBpIsHwIoArmed(pVM)))
11173 {
11174 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
11175
11176 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
11177 VMMRZCallRing3Disable(pVCpu);
11178 HM_DISABLE_PREEMPT_IF_NEEDED();
11179
11180 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
11181
11182 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
11183 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
11184 {
11185 /* Raise #DB. */
11186 if (fIsGuestDbgActive)
11187 ASMSetDR6(pMixedCtx->dr[6]);
11188 if (pMixedCtx->dr[7] != uDr7)
11189 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11190
11191 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
11192 }
11193 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
11194 else if ( rcStrict2 != VINF_SUCCESS
11195 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
11196 rcStrict = rcStrict2;
11197
11198 HM_RESTORE_PREEMPT_IF_NEEDED();
11199 VMMRZCallRing3Enable(pVCpu);
11200 }
11201 }
11202
11203#ifdef DEBUG
11204 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11205 Assert(!fIOWrite);
11206 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11207 Assert(fIOWrite);
11208 else
11209 {
11210 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
11211 * statuses, that the VMM device and some others may return. See
11212 * IOM_SUCCESS() for guidance. */
11213 AssertMsg( RT_FAILURE(rcStrict)
11214 || rcStrict == VINF_SUCCESS
11215 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
11216 || rcStrict == VINF_EM_DBG_BREAKPOINT
11217 || rcStrict == VINF_EM_RAW_GUEST_TRAP
11218 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11219 }
11220#endif
11221
11222 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
11223 return VBOXSTRICTRC_TODO(rcStrict);
11224}
11225
11226
11227/**
11228 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
11229 * VM-exit.
11230 */
11231HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11232{
11233 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11234
11235 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
11236 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11237 AssertRCReturn(rc, rc);
11238 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
11239 {
11240 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
11241 AssertRCReturn(rc, rc);
11242 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
11243 {
11244 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
11245
11246 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
11247 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
11248
11249 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
11250 Assert(!pVCpu->hm.s.Event.fPending);
11251 pVCpu->hm.s.Event.fPending = true;
11252 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
11253 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
11254 AssertRCReturn(rc, rc);
11255 if (fErrorCodeValid)
11256 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
11257 else
11258 pVCpu->hm.s.Event.u32ErrCode = 0;
11259 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
11260 && uVector == X86_XCPT_PF)
11261 {
11262 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
11263 }
11264
11265 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
11266 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11267 return VINF_EM_RAW_INJECT_TRPM_EVENT;
11268 }
11269 }
11270
11271 /** @todo Emulate task switch someday, currently just going back to ring-3 for
11272 * emulation. */
11273 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11274 return VERR_EM_INTERPRETER;
11275}
11276
11277
11278/**
11279 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
11280 */
11281HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11282{
11283 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11284 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
11285 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
11286 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11287 AssertRCReturn(rc, rc);
11288 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
11289 return VINF_EM_DBG_STEPPED;
11290}
11291
11292
11293/**
11294 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
11295 */
11296HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11297{
11298 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11299
11300 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11301 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11302 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11303 {
11304 if (rc == VINF_HM_DOUBLE_FAULT)
11305 rc = VINF_SUCCESS;
11306 return rc;
11307 }
11308
11309#if 0
11310 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
11311 * just sync the whole thing. */
11312 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11313#else
11314 /* Aggressive state sync. for now. */
11315 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11316 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11317 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11318#endif
11319 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11320 AssertRCReturn(rc, rc);
11321
11322 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
11323 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
11324 switch (uAccessType)
11325 {
11326 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
11327 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
11328 {
11329 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
11330 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != 0x80,
11331 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
11332
11333 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
11334 GCPhys &= PAGE_BASE_GC_MASK;
11335 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
11336 PVM pVM = pVCpu->CTX_SUFF(pVM);
11337 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
11338 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
11339
11340 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
11341 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
11342 CPUMCTX2CORE(pMixedCtx), GCPhys);
11343 rc = VBOXSTRICTRC_VAL(rc2);
11344 Log4(("ApicAccess rc=%d\n", rc));
11345 if ( rc == VINF_SUCCESS
11346 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11347 || rc == VERR_PAGE_NOT_PRESENT)
11348 {
11349 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11350 | HM_CHANGED_GUEST_RSP
11351 | HM_CHANGED_GUEST_RFLAGS
11352 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11353 rc = VINF_SUCCESS;
11354 }
11355 break;
11356 }
11357
11358 default:
11359 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
11360 rc = VINF_EM_RAW_EMULATE_INSTR;
11361 break;
11362 }
11363
11364 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
11365 if (rc != VINF_SUCCESS)
11366 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
11367 return rc;
11368}
11369
11370
11371/**
11372 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
11373 * VM-exit.
11374 */
11375HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11376{
11377 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11378
11379 /* We should -not- get this VM-exit if the guest's debug registers were active. */
11380 if (pVmxTransient->fWasGuestDebugStateActive)
11381 {
11382 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11383 HMVMX_RETURN_UNEXPECTED_EXIT();
11384 }
11385
11386 int rc = VERR_INTERNAL_ERROR_5;
11387 if ( !DBGFIsStepping(pVCpu)
11388 && !pVCpu->hm.s.fSingleInstruction
11389 && !pVmxTransient->fWasHyperDebugStateActive)
11390 {
11391 /* Don't intercept MOV DRx and #DB any more. */
11392 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
11393 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11394 AssertRCReturn(rc, rc);
11395
11396 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11397 {
11398#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11399 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
11400 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
11401 AssertRCReturn(rc, rc);
11402#endif
11403 }
11404
11405 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
11406 VMMRZCallRing3Disable(pVCpu);
11407 HM_DISABLE_PREEMPT_IF_NEEDED();
11408
11409 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
11410 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
11411 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
11412
11413 HM_RESTORE_PREEMPT_IF_NEEDED();
11414 VMMRZCallRing3Enable(pVCpu);
11415
11416#ifdef VBOX_WITH_STATISTICS
11417 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11418 AssertRCReturn(rc, rc);
11419 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11420 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11421 else
11422 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11423#endif
11424 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
11425 return VINF_SUCCESS;
11426 }
11427
11428 /*
11429 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
11430 * Update the segment registers and DR7 from the CPU.
11431 */
11432 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11433 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11434 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11435 AssertRCReturn(rc, rc);
11436 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11437
11438 PVM pVM = pVCpu->CTX_SUFF(pVM);
11439 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11440 {
11441 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11442 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
11443 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
11444 if (RT_SUCCESS(rc))
11445 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11446 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11447 }
11448 else
11449 {
11450 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11451 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
11452 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
11453 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11454 }
11455
11456 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
11457 if (RT_SUCCESS(rc))
11458 {
11459 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11460 AssertRCReturn(rc2, rc2);
11461 }
11462 return rc;
11463}
11464
11465
11466/**
11467 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
11468 * Conditional VM-exit.
11469 */
11470HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11471{
11472 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11473 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11474
11475 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11476 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11477 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11478 {
11479 if (rc == VINF_HM_DOUBLE_FAULT)
11480 rc = VINF_SUCCESS;
11481 return rc;
11482 }
11483
11484 RTGCPHYS GCPhys = 0;
11485 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11486
11487#if 0
11488 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11489#else
11490 /* Aggressive state sync. for now. */
11491 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11492 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11493 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11494#endif
11495 AssertRCReturn(rc, rc);
11496
11497 /*
11498 * If we succeed, resume guest execution.
11499 * If we fail in interpreting the instruction because we couldn't get the guest physical address
11500 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
11501 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
11502 * weird case. See @bugref{6043}.
11503 */
11504 PVM pVM = pVCpu->CTX_SUFF(pVM);
11505 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
11506 rc = VBOXSTRICTRC_VAL(rc2);
11507 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
11508 if ( rc == VINF_SUCCESS
11509 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11510 || rc == VERR_PAGE_NOT_PRESENT)
11511 {
11512 /* Successfully handled MMIO operation. */
11513 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11514 | HM_CHANGED_GUEST_RSP
11515 | HM_CHANGED_GUEST_RFLAGS
11516 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11517 rc = VINF_SUCCESS;
11518 }
11519 return rc;
11520}
11521
11522
11523/**
11524 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
11525 * VM-exit.
11526 */
11527HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11528{
11529 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11530 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11531
11532 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11533 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11534 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11535 {
11536 if (rc == VINF_HM_DOUBLE_FAULT)
11537 rc = VINF_SUCCESS;
11538 return rc;
11539 }
11540
11541 RTGCPHYS GCPhys = 0;
11542 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11543 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11544#if 0
11545 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11546#else
11547 /* Aggressive state sync. for now. */
11548 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11549 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11550 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11551#endif
11552 AssertRCReturn(rc, rc);
11553
11554 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
11555 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
11556
11557 RTGCUINT uErrorCode = 0;
11558 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
11559 uErrorCode |= X86_TRAP_PF_ID;
11560 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
11561 uErrorCode |= X86_TRAP_PF_RW;
11562 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
11563 uErrorCode |= X86_TRAP_PF_P;
11564
11565 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
11566
11567 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
11568 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11569
11570 /* Handle the pagefault trap for the nested shadow table. */
11571 PVM pVM = pVCpu->CTX_SUFF(pVM);
11572 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
11573 TRPMResetTrap(pVCpu);
11574
11575 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
11576 if ( rc == VINF_SUCCESS
11577 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11578 || rc == VERR_PAGE_NOT_PRESENT)
11579 {
11580 /* Successfully synced our nested page tables. */
11581 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
11582 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11583 | HM_CHANGED_GUEST_RSP
11584 | HM_CHANGED_GUEST_RFLAGS);
11585 return VINF_SUCCESS;
11586 }
11587
11588 Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
11589 return rc;
11590}
11591
11592/** @} */
11593
11594/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11595/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
11596/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11597
11598/** @name VM-exit exception handlers.
11599 * @{
11600 */
11601
11602/**
11603 * VM-exit exception handler for #MF (Math Fault: floating point exception).
11604 */
11605static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11606{
11607 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11608 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
11609
11610 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11611 AssertRCReturn(rc, rc);
11612
11613 if (!(pMixedCtx->cr0 & X86_CR0_NE))
11614 {
11615 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
11616 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
11617
11618 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
11619 * provides VM-exit instruction length. If this causes problem later,
11620 * disassemble the instruction like it's done on AMD-V. */
11621 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11622 AssertRCReturn(rc2, rc2);
11623 return rc;
11624 }
11625
11626 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11627 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11628 return rc;
11629}
11630
11631
11632/**
11633 * VM-exit exception handler for #BP (Breakpoint exception).
11634 */
11635static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11636{
11637 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11638 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
11639
11640 /** @todo Try optimize this by not saving the entire guest state unless
11641 * really needed. */
11642 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11643 AssertRCReturn(rc, rc);
11644
11645 PVM pVM = pVCpu->CTX_SUFF(pVM);
11646 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11647 if (rc == VINF_EM_RAW_GUEST_TRAP)
11648 {
11649 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11650 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11651 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11652 AssertRCReturn(rc, rc);
11653
11654 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11655 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11656 }
11657
11658 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
11659 return rc;
11660}
11661
11662
11663/**
11664 * VM-exit exception handler for #DB (Debug exception).
11665 */
11666static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11667{
11668 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11669 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
11670 Log6(("XcptDB\n"));
11671
11672 /*
11673 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
11674 * for processing.
11675 */
11676 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11677 AssertRCReturn(rc, rc);
11678
11679 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
11680 uint64_t uDR6 = X86_DR6_INIT_VAL;
11681 uDR6 |= ( pVmxTransient->uExitQualification
11682 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
11683
11684 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
11685 if (rc == VINF_EM_RAW_GUEST_TRAP)
11686 {
11687 /*
11688 * The exception was for the guest. Update DR6, DR7.GD and
11689 * IA32_DEBUGCTL.LBR before forwarding it.
11690 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
11691 */
11692 VMMRZCallRing3Disable(pVCpu);
11693 HM_DISABLE_PREEMPT_IF_NEEDED();
11694
11695 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
11696 pMixedCtx->dr[6] |= uDR6;
11697 if (CPUMIsGuestDebugStateActive(pVCpu))
11698 ASMSetDR6(pMixedCtx->dr[6]);
11699
11700 HM_RESTORE_PREEMPT_IF_NEEDED();
11701 VMMRZCallRing3Enable(pVCpu);
11702
11703 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11704 AssertRCReturn(rc, rc);
11705
11706 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
11707 pMixedCtx->dr[7] &= ~X86_DR7_GD;
11708
11709 /* Paranoia. */
11710 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
11711 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
11712
11713 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
11714 AssertRCReturn(rc, rc);
11715
11716 /*
11717 * Raise #DB in the guest.
11718 *
11719 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
11720 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
11721 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
11722 *
11723 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
11724 */
11725 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11726 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11727 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11728 AssertRCReturn(rc, rc);
11729 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11730 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11731 return VINF_SUCCESS;
11732 }
11733
11734 /*
11735 * Not a guest trap, must be a hypervisor related debug event then.
11736 * Update DR6 in case someone is interested in it.
11737 */
11738 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
11739 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
11740 CPUMSetHyperDR6(pVCpu, uDR6);
11741
11742 return rc;
11743}
11744
11745
11746/**
11747 * VM-exit exception handler for #NM (Device-not-available exception: floating
11748 * point exception).
11749 */
11750static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11751{
11752 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11753
11754 /* We require CR0 and EFER. EFER is always up-to-date. */
11755 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11756 AssertRCReturn(rc, rc);
11757
11758 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11759 VMMRZCallRing3Disable(pVCpu);
11760 HM_DISABLE_PREEMPT_IF_NEEDED();
11761
11762 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11763 if (pVmxTransient->fWasGuestFPUStateActive)
11764 {
11765 rc = VINF_EM_RAW_GUEST_TRAP;
11766 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11767 }
11768 else
11769 {
11770#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11771 Assert(!pVmxTransient->fWasGuestFPUStateActive);
11772#endif
11773 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11774 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11775 }
11776
11777 HM_RESTORE_PREEMPT_IF_NEEDED();
11778 VMMRZCallRing3Enable(pVCpu);
11779
11780 if (rc == VINF_SUCCESS)
11781 {
11782 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11783 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11784 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11785 pVCpu->hm.s.fUseGuestFpu = true;
11786 }
11787 else
11788 {
11789 /* Forward #NM to the guest. */
11790 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11791 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11792 AssertRCReturn(rc, rc);
11793 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11794 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
11795 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11796 }
11797
11798 return VINF_SUCCESS;
11799}
11800
11801
11802/**
11803 * VM-exit exception handler for #GP (General-protection exception).
11804 *
11805 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11806 */
11807static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11808{
11809 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11810 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11811
11812 int rc = VERR_INTERNAL_ERROR_5;
11813 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11814 {
11815#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11816 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11817 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11818 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11819 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11820 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11821 AssertRCReturn(rc, rc);
11822 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
11823 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
11824 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11825 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11826 return rc;
11827#else
11828 /* We don't intercept #GP. */
11829 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11830 NOREF(pVmxTransient);
11831 return VERR_VMX_UNEXPECTED_EXCEPTION;
11832#endif
11833 }
11834
11835 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11836 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11837
11838 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11839 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11840 AssertRCReturn(rc, rc);
11841
11842 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11843 uint32_t cbOp = 0;
11844 PVM pVM = pVCpu->CTX_SUFF(pVM);
11845 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11846 if (RT_SUCCESS(rc))
11847 {
11848 rc = VINF_SUCCESS;
11849 Assert(cbOp == pDis->cbInstr);
11850 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11851 switch (pDis->pCurInstr->uOpcode)
11852 {
11853 case OP_CLI:
11854 {
11855 pMixedCtx->eflags.Bits.u1IF = 0;
11856 pMixedCtx->eflags.Bits.u1RF = 0;
11857 pMixedCtx->rip += pDis->cbInstr;
11858 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11859 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11860 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11861 break;
11862 }
11863
11864 case OP_STI:
11865 {
11866 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
11867 pMixedCtx->eflags.Bits.u1IF = 1;
11868 pMixedCtx->eflags.Bits.u1RF = 0;
11869 pMixedCtx->rip += pDis->cbInstr;
11870 if (!fOldIF)
11871 {
11872 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11873 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11874 }
11875 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11876 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11877 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11878 break;
11879 }
11880
11881 case OP_HLT:
11882 {
11883 rc = VINF_EM_HALT;
11884 pMixedCtx->rip += pDis->cbInstr;
11885 pMixedCtx->eflags.Bits.u1RF = 0;
11886 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11887 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11888 break;
11889 }
11890
11891 case OP_POPF:
11892 {
11893 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11894 uint32_t cbParm;
11895 uint32_t uMask;
11896 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11897 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11898 {
11899 cbParm = 4;
11900 uMask = 0xffffffff;
11901 }
11902 else
11903 {
11904 cbParm = 2;
11905 uMask = 0xffff;
11906 }
11907
11908 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
11909 RTGCPTR GCPtrStack = 0;
11910 X86EFLAGS Eflags;
11911 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11912 &GCPtrStack);
11913 if (RT_SUCCESS(rc))
11914 {
11915 Assert(sizeof(Eflags.u32) >= cbParm);
11916 Eflags.u32 = 0;
11917 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11918 }
11919 if (RT_FAILURE(rc))
11920 {
11921 rc = VERR_EM_INTERPRETER;
11922 break;
11923 }
11924 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11925 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
11926 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11927 pMixedCtx->esp += cbParm;
11928 pMixedCtx->esp &= uMask;
11929 pMixedCtx->rip += pDis->cbInstr;
11930 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11931 | HM_CHANGED_GUEST_RSP
11932 | HM_CHANGED_GUEST_RFLAGS);
11933 /* Generate a pending-debug exception when stepping over POPF regardless of how POPF modifies EFLAGS.TF. */
11934 if (fStepping)
11935 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11936
11937 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11938 break;
11939 }
11940
11941 case OP_PUSHF:
11942 {
11943 uint32_t cbParm;
11944 uint32_t uMask;
11945 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11946 {
11947 cbParm = 4;
11948 uMask = 0xffffffff;
11949 }
11950 else
11951 {
11952 cbParm = 2;
11953 uMask = 0xffff;
11954 }
11955
11956 /* Get the stack pointer & push the contents of eflags onto the stack. */
11957 RTGCPTR GCPtrStack = 0;
11958 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11959 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11960 if (RT_FAILURE(rc))
11961 {
11962 rc = VERR_EM_INTERPRETER;
11963 break;
11964 }
11965 X86EFLAGS Eflags = pMixedCtx->eflags;
11966 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11967 Eflags.Bits.u1RF = 0;
11968 Eflags.Bits.u1VM = 0;
11969
11970 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11971 if (RT_FAILURE(rc))
11972 {
11973 rc = VERR_EM_INTERPRETER;
11974 break;
11975 }
11976 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11977 pMixedCtx->esp -= cbParm;
11978 pMixedCtx->esp &= uMask;
11979 pMixedCtx->rip += pDis->cbInstr;
11980 pMixedCtx->eflags.Bits.u1RF = 0;
11981 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11982 | HM_CHANGED_GUEST_RSP
11983 | HM_CHANGED_GUEST_RFLAGS);
11984 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11985 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11986 break;
11987 }
11988
11989 case OP_IRET:
11990 {
11991 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11992 * instruction reference. */
11993 RTGCPTR GCPtrStack = 0;
11994 uint32_t uMask = 0xffff;
11995 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11996 uint16_t aIretFrame[3];
11997 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
11998 {
11999 rc = VERR_EM_INTERPRETER;
12000 break;
12001 }
12002 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
12003 &GCPtrStack);
12004 if (RT_SUCCESS(rc))
12005 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
12006 if (RT_FAILURE(rc))
12007 {
12008 rc = VERR_EM_INTERPRETER;
12009 break;
12010 }
12011 pMixedCtx->eip = 0;
12012 pMixedCtx->ip = aIretFrame[0];
12013 pMixedCtx->cs.Sel = aIretFrame[1];
12014 pMixedCtx->cs.ValidSel = aIretFrame[1];
12015 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
12016 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
12017 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
12018 pMixedCtx->sp += sizeof(aIretFrame);
12019 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12020 | HM_CHANGED_GUEST_SEGMENT_REGS
12021 | HM_CHANGED_GUEST_RSP
12022 | HM_CHANGED_GUEST_RFLAGS);
12023 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
12024 if (fStepping)
12025 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
12026 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
12027 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
12028 break;
12029 }
12030
12031 case OP_INT:
12032 {
12033 uint16_t uVector = pDis->Param1.uValue & 0xff;
12034 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
12035 /* INT clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
12036 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
12037 break;
12038 }
12039
12040 case OP_INTO:
12041 {
12042 if (pMixedCtx->eflags.Bits.u1OF)
12043 {
12044 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
12045 /* INTO clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
12046 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
12047 }
12048 else
12049 {
12050 pMixedCtx->eflags.Bits.u1RF = 0;
12051 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12052 }
12053 break;
12054 }
12055
12056 default:
12057 {
12058 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
12059 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
12060 EMCODETYPE_SUPERVISOR);
12061 rc = VBOXSTRICTRC_VAL(rc2);
12062 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
12063 /** @todo We have to set pending-debug exceptions here when the guest is
12064 * single-stepping depending on the instruction that was interpreted. */
12065 Log4(("#GP rc=%Rrc\n", rc));
12066 break;
12067 }
12068 }
12069 }
12070 else
12071 rc = VERR_EM_INTERPRETER;
12072
12073 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
12074 ("#GP Unexpected rc=%Rrc\n", rc));
12075 return rc;
12076}
12077
12078
12079#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12080/**
12081 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
12082 * the exception reported in the VMX transient structure back into the VM.
12083 *
12084 * @remarks Requires uExitIntInfo in the VMX transient structure to be
12085 * up-to-date.
12086 */
12087static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12088{
12089 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12090
12091 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
12092 hmR0VmxCheckExitDueToEventDelivery(). */
12093 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12094 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12095 AssertRCReturn(rc, rc);
12096 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
12097
12098#ifdef DEBUG_ramshankar
12099 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12100 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12101 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
12102#endif
12103
12104 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12105 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12106 return VINF_SUCCESS;
12107}
12108#endif
12109
12110
12111/**
12112 * VM-exit exception handler for #PF (Page-fault exception).
12113 */
12114static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12115{
12116 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12117 PVM pVM = pVCpu->CTX_SUFF(pVM);
12118 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12119 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12120 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12121 AssertRCReturn(rc, rc);
12122
12123#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
12124 if (pVM->hm.s.fNestedPaging)
12125 {
12126 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
12127 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
12128 {
12129 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12130 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12131 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
12132 }
12133 else
12134 {
12135 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12136 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12137 Log4(("Pending #DF due to vectoring #PF. NP\n"));
12138 }
12139 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12140 return rc;
12141 }
12142#else
12143 Assert(!pVM->hm.s.fNestedPaging);
12144 NOREF(pVM);
12145#endif
12146
12147 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
12148 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
12149 if (pVmxTransient->fVectoringPF)
12150 {
12151 Assert(pVCpu->hm.s.Event.fPending);
12152 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12153 }
12154
12155 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12156 AssertRCReturn(rc, rc);
12157
12158 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
12159 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
12160
12161 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
12162 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
12163 (RTGCPTR)pVmxTransient->uExitQualification);
12164
12165 Log4(("#PF: rc=%Rrc\n", rc));
12166 if (rc == VINF_SUCCESS)
12167 {
12168 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
12169 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
12170 * memory? We don't update the whole state here... */
12171 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12172 | HM_CHANGED_GUEST_RSP
12173 | HM_CHANGED_GUEST_RFLAGS
12174 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12175 TRPMResetTrap(pVCpu);
12176 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
12177 return rc;
12178 }
12179
12180 if (rc == VINF_EM_RAW_GUEST_TRAP)
12181 {
12182 if (!pVmxTransient->fVectoringDoublePF)
12183 {
12184 /* It's a guest page fault and needs to be reflected to the guest. */
12185 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
12186 TRPMResetTrap(pVCpu);
12187 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
12188 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12189 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12190 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
12191 }
12192 else
12193 {
12194 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12195 TRPMResetTrap(pVCpu);
12196 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
12197 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12198 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
12199 }
12200
12201 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12202 return VINF_SUCCESS;
12203 }
12204
12205 TRPMResetTrap(pVCpu);
12206 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
12207 return rc;
12208}
12209
12210/** @} */
12211
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette