VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 54650

Last change on this file since 54650 was 54650, checked in by vboxsync, 10 years ago

SUPDrv/VMMR0: properly handle the CR4 shadow register on Linux >= 4.0
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 506.3 KB
Line 
1/* $Id: HMVMXR0.cpp 54650 2015-03-05 13:42:10Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2015 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/x86.h>
23#include <iprt/asm-amd64-x86.h>
24#include <iprt/thread.h>
25
26#include "HMInternal.h"
27#include <VBox/vmm/vm.h>
28#include "HMVMXR0.h"
29#include <VBox/vmm/pdmapi.h>
30#include <VBox/vmm/dbgf.h>
31#include <VBox/vmm/iem.h>
32#include <VBox/vmm/iom.h>
33#include <VBox/vmm/selm.h>
34#include <VBox/vmm/tm.h>
35#include <VBox/vmm/gim.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39# include "dtrace/VBoxVMM.h"
40
41#ifdef DEBUG_ramshankar
42# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
43# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
44# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
45# define HMVMX_ALWAYS_CHECK_GUEST_STATE
46# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
47# define HMVMX_ALWAYS_TRAP_PF
48# define HMVMX_ALWAYS_SWAP_FPU_STATE
49# define HMVMX_ALWAYS_FLUSH_TLB
50# define HMVMX_ALWAYS_SWAP_EFER
51#endif
52
53
54/*******************************************************************************
55* Defined Constants And Macros *
56*******************************************************************************/
57#if defined(RT_ARCH_AMD64)
58# define HMVMX_IS_64BIT_HOST_MODE() (true)
59typedef RTHCUINTREG HMVMXHCUINTREG;
60#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
61extern "C" uint32_t g_fVMXIs64bitHost;
62# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
63typedef uint64_t HMVMXHCUINTREG;
64#else
65# define HMVMX_IS_64BIT_HOST_MODE() (false)
66typedef RTHCUINTREG HMVMXHCUINTREG;
67#endif
68
69/** Use the function table. */
70#define HMVMX_USE_FUNCTION_TABLE
71
72/** Determine which tagged-TLB flush handler to use. */
73#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
74#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
75#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
76#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
77
78/** @name Updated-guest-state flags.
79 * @{ */
80#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
81#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
82#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
83#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
84#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
85#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
86#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
87#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
88#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
89#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
90#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
91#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
92#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
93#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
94#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
95#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
96#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
97#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
98#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
99#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
100#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
101 | HMVMX_UPDATED_GUEST_RSP \
102 | HMVMX_UPDATED_GUEST_RFLAGS \
103 | HMVMX_UPDATED_GUEST_CR0 \
104 | HMVMX_UPDATED_GUEST_CR3 \
105 | HMVMX_UPDATED_GUEST_CR4 \
106 | HMVMX_UPDATED_GUEST_GDTR \
107 | HMVMX_UPDATED_GUEST_IDTR \
108 | HMVMX_UPDATED_GUEST_LDTR \
109 | HMVMX_UPDATED_GUEST_TR \
110 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
111 | HMVMX_UPDATED_GUEST_DEBUG \
112 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
113 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
114 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
115 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
116 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
117 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
118 | HMVMX_UPDATED_GUEST_INTR_STATE \
119 | HMVMX_UPDATED_GUEST_APIC_STATE)
120/** @} */
121
122/** @name
123 * Flags to skip redundant reads of some common VMCS fields that are not part of
124 * the guest-CPU state but are in the transient structure.
125 */
126#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
127#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
128#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
129#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
130#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
131#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
132#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
133/** @} */
134
135/** @name
136 * States of the VMCS.
137 *
138 * This does not reflect all possible VMCS states but currently only those
139 * needed for maintaining the VMCS consistently even when thread-context hooks
140 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
141 */
142#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
143#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
144#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
145/** @} */
146
147/**
148 * Exception bitmap mask for real-mode guests (real-on-v86).
149 *
150 * We need to intercept all exceptions manually except:
151 * - #NM, #MF handled in hmR0VmxLoadSharedCR0().
152 * - #DB handled in hmR0VmxLoadSharedDebugState().
153 * - #PF need not be intercepted even in real-mode if we have Nested Paging
154 * support.
155 */
156#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
157 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
158 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
159 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
160 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
161 /* RT_BIT(X86_XCPT_MF) */ | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
162 | RT_BIT(X86_XCPT_XF))
163
164/**
165 * Exception bitmap mask for all contributory exceptions.
166 *
167 * Page fault is deliberately excluded here as it's conditional as to whether
168 * it's contributory or benign. Page faults are handled separately.
169 */
170#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
171 | RT_BIT(X86_XCPT_DE))
172
173/** Maximum VM-instruction error number. */
174#define HMVMX_INSTR_ERROR_MAX 28
175
176/** Profiling macro. */
177#ifdef HM_PROFILE_EXIT_DISPATCH
178# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
179# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
180#else
181# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
182# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
183#endif
184
185/** Assert that preemption is disabled or covered by thread-context hooks. */
186#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
187 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
188
189/** Assert that we haven't migrated CPUs when thread-context hooks are not
190 * used. */
191#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
192 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
193 ("Illegal migration! Entered on CPU %u Current %u\n", \
194 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
195
196/** Helper macro for VM-exit handlers called unexpectedly. */
197#define HMVMX_RETURN_UNEXPECTED_EXIT() \
198 do { \
199 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
200 return VERR_VMX_UNEXPECTED_EXIT; \
201 } while (0)
202
203
204/*******************************************************************************
205* Structures and Typedefs *
206*******************************************************************************/
207/**
208 * VMX transient state.
209 *
210 * A state structure for holding miscellaneous information across
211 * VMX non-root operation and restored after the transition.
212 */
213typedef struct VMXTRANSIENT
214{
215 /** The host's rflags/eflags. */
216 RTCCUINTREG uEflags;
217#if HC_ARCH_BITS == 32
218 uint32_t u32Alignment0;
219#endif
220 /** The guest's TPR value used for TPR shadowing. */
221 uint8_t u8GuestTpr;
222 /** Alignment. */
223 uint8_t abAlignment0[7];
224
225 /** The basic VM-exit reason. */
226 uint16_t uExitReason;
227 /** Alignment. */
228 uint16_t u16Alignment0;
229 /** The VM-exit interruption error code. */
230 uint32_t uExitIntErrorCode;
231 /** The VM-exit exit code qualification. */
232 uint64_t uExitQualification;
233
234 /** The VM-exit interruption-information field. */
235 uint32_t uExitIntInfo;
236 /** The VM-exit instruction-length field. */
237 uint32_t cbInstr;
238 /** The VM-exit instruction-information field. */
239 union
240 {
241 /** Plain unsigned int representation. */
242 uint32_t u;
243 /** INS and OUTS information. */
244 struct
245 {
246 uint32_t u6Reserved0 : 7;
247 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
248 uint32_t u3AddrSize : 3;
249 uint32_t u5Reserved1 : 5;
250 /** The segment register (X86_SREG_XXX). */
251 uint32_t iSegReg : 3;
252 uint32_t uReserved2 : 14;
253 } StrIo;
254 } ExitInstrInfo;
255 /** Whether the VM-entry failed or not. */
256 bool fVMEntryFailed;
257 /** Alignment. */
258 uint8_t abAlignment1[3];
259
260 /** The VM-entry interruption-information field. */
261 uint32_t uEntryIntInfo;
262 /** The VM-entry exception error code field. */
263 uint32_t uEntryXcptErrorCode;
264 /** The VM-entry instruction length field. */
265 uint32_t cbEntryInstr;
266
267 /** IDT-vectoring information field. */
268 uint32_t uIdtVectoringInfo;
269 /** IDT-vectoring error code. */
270 uint32_t uIdtVectoringErrorCode;
271
272 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
273 uint32_t fVmcsFieldsRead;
274
275 /** Whether the guest FPU was active at the time of VM-exit. */
276 bool fWasGuestFPUStateActive;
277 /** Whether the guest debug state was active at the time of VM-exit. */
278 bool fWasGuestDebugStateActive;
279 /** Whether the hyper debug state was active at the time of VM-exit. */
280 bool fWasHyperDebugStateActive;
281 /** Whether TSC-offsetting should be setup before VM-entry. */
282 bool fUpdateTscOffsettingAndPreemptTimer;
283 /** Whether the VM-exit was caused by a page-fault during delivery of a
284 * contributory exception or a page-fault. */
285 bool fVectoringDoublePF;
286 /** Whether the VM-exit was caused by a page-fault during delivery of an
287 * external interrupt or NMI. */
288 bool fVectoringPF;
289} VMXTRANSIENT;
290AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
291AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
292AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
293AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
294AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
295/** Pointer to VMX transient state. */
296typedef VMXTRANSIENT *PVMXTRANSIENT;
297
298
299/**
300 * MSR-bitmap read permissions.
301 */
302typedef enum VMXMSREXITREAD
303{
304 /** Reading this MSR causes a VM-exit. */
305 VMXMSREXIT_INTERCEPT_READ = 0xb,
306 /** Reading this MSR does not cause a VM-exit. */
307 VMXMSREXIT_PASSTHRU_READ
308} VMXMSREXITREAD;
309/** Pointer to MSR-bitmap read permissions. */
310typedef VMXMSREXITREAD* PVMXMSREXITREAD;
311
312/**
313 * MSR-bitmap write permissions.
314 */
315typedef enum VMXMSREXITWRITE
316{
317 /** Writing to this MSR causes a VM-exit. */
318 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
319 /** Writing to this MSR does not cause a VM-exit. */
320 VMXMSREXIT_PASSTHRU_WRITE
321} VMXMSREXITWRITE;
322/** Pointer to MSR-bitmap write permissions. */
323typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
324
325
326/**
327 * VMX VM-exit handler.
328 *
329 * @returns VBox status code.
330 * @param pVCpu Pointer to the VMCPU.
331 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
332 * out-of-sync. Make sure to update the required
333 * fields before using them.
334 * @param pVmxTransient Pointer to the VMX-transient structure.
335 */
336#ifndef HMVMX_USE_FUNCTION_TABLE
337typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
338#else
339typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
340/** Pointer to VM-exit handler. */
341typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
342#endif
343
344
345/*******************************************************************************
346* Internal Functions *
347*******************************************************************************/
348static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
349static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
350static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
351 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
352 bool fStepping, uint32_t *puIntState);
353#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
354static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
355#endif
356#ifndef HMVMX_USE_FUNCTION_TABLE
357DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
358# define HMVMX_EXIT_DECL static int
359#else
360# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
361#endif
362DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitStep(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
363 uint32_t uExitReason, uint16_t uCsStart, uint64_t uRipStart);
364
365/** @name VM-exit handlers.
366 * @{
367 */
368static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
369static FNVMXEXITHANDLER hmR0VmxExitExtInt;
370static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
371static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
372static FNVMXEXITHANDLER hmR0VmxExitSipi;
373static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
374static FNVMXEXITHANDLER hmR0VmxExitSmi;
375static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
376static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
377static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
378static FNVMXEXITHANDLER hmR0VmxExitCpuid;
379static FNVMXEXITHANDLER hmR0VmxExitGetsec;
380static FNVMXEXITHANDLER hmR0VmxExitHlt;
381static FNVMXEXITHANDLER hmR0VmxExitInvd;
382static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
383static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
384static FNVMXEXITHANDLER hmR0VmxExitVmcall;
385static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
386static FNVMXEXITHANDLER hmR0VmxExitRsm;
387static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
388static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
389static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
390static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
391static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
392static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
393static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
394static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
395static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
396static FNVMXEXITHANDLER hmR0VmxExitMwait;
397static FNVMXEXITHANDLER hmR0VmxExitMtf;
398static FNVMXEXITHANDLER hmR0VmxExitMonitor;
399static FNVMXEXITHANDLER hmR0VmxExitPause;
400static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
401static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
402static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
403static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
404static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
405static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
406static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
407static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
408static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
409static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
410static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
411static FNVMXEXITHANDLER hmR0VmxExitRdrand;
412static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
413/** @} */
414
415static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
416static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
417static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
418static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
419static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
420static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
421#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
422static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
423#endif
424static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
425
426/*******************************************************************************
427* Global Variables *
428*******************************************************************************/
429#ifdef HMVMX_USE_FUNCTION_TABLE
430
431/**
432 * VMX_EXIT dispatch table.
433 */
434static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
435{
436 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
437 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
438 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
439 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
440 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
441 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
442 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
443 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
444 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
445 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
446 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
447 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
448 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
449 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
450 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
451 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
452 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
453 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
454 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
455 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
456 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
457 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
458 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
459 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
460 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
461 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
462 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
463 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
464 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
465 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
466 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
467 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
468 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
469 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
470 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
471 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
472 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
473 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
474 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
475 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
476 /* 40 UNDEFINED */ hmR0VmxExitPause,
477 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
478 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
479 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
480 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
481 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
482 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
483 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
484 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
485 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
486 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
487 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
488 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
489 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
490 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
491 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
492 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
493 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
494 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
495 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
496};
497#endif /* HMVMX_USE_FUNCTION_TABLE */
498
499#ifdef VBOX_STRICT
500static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
501{
502 /* 0 */ "(Not Used)",
503 /* 1 */ "VMCALL executed in VMX root operation.",
504 /* 2 */ "VMCLEAR with invalid physical address.",
505 /* 3 */ "VMCLEAR with VMXON pointer.",
506 /* 4 */ "VMLAUNCH with non-clear VMCS.",
507 /* 5 */ "VMRESUME with non-launched VMCS.",
508 /* 6 */ "VMRESUME after VMXOFF",
509 /* 7 */ "VM-entry with invalid control fields.",
510 /* 8 */ "VM-entry with invalid host state fields.",
511 /* 9 */ "VMPTRLD with invalid physical address.",
512 /* 10 */ "VMPTRLD with VMXON pointer.",
513 /* 11 */ "VMPTRLD with incorrect revision identifier.",
514 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
515 /* 13 */ "VMWRITE to read-only VMCS component.",
516 /* 14 */ "(Not Used)",
517 /* 15 */ "VMXON executed in VMX root operation.",
518 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
519 /* 17 */ "VM-entry with non-launched executing VMCS.",
520 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
521 /* 19 */ "VMCALL with non-clear VMCS.",
522 /* 20 */ "VMCALL with invalid VM-exit control fields.",
523 /* 21 */ "(Not Used)",
524 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
525 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
526 /* 24 */ "VMCALL with invalid SMM-monitor features.",
527 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
528 /* 26 */ "VM-entry with events blocked by MOV SS.",
529 /* 27 */ "(Not Used)",
530 /* 28 */ "Invalid operand to INVEPT/INVVPID."
531};
532#endif /* VBOX_STRICT */
533
534
535
536/**
537 * Updates the VM's last error record. If there was a VMX instruction error,
538 * reads the error data from the VMCS and updates VCPU's last error record as
539 * well.
540 *
541 * @param pVM Pointer to the VM.
542 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
543 * VERR_VMX_UNABLE_TO_START_VM or
544 * VERR_VMX_INVALID_VMCS_FIELD).
545 * @param rc The error code.
546 */
547static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
548{
549 AssertPtr(pVM);
550 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
551 || rc == VERR_VMX_UNABLE_TO_START_VM)
552 {
553 AssertPtrReturnVoid(pVCpu);
554 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
555 }
556 pVM->hm.s.lLastError = rc;
557}
558
559
560/**
561 * Reads the VM-entry interruption-information field from the VMCS into the VMX
562 * transient structure.
563 *
564 * @returns VBox status code.
565 * @param pVmxTransient Pointer to the VMX transient structure.
566 *
567 * @remarks No-long-jump zone!!!
568 */
569DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
570{
571 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
572 AssertRCReturn(rc, rc);
573 return VINF_SUCCESS;
574}
575
576
577/**
578 * Reads the VM-entry exception error code field from the VMCS into
579 * the VMX transient structure.
580 *
581 * @returns VBox status code.
582 * @param pVmxTransient Pointer to the VMX transient structure.
583 *
584 * @remarks No-long-jump zone!!!
585 */
586DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
587{
588 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
589 AssertRCReturn(rc, rc);
590 return VINF_SUCCESS;
591}
592
593
594/**
595 * Reads the VM-entry exception error code field from the VMCS into
596 * the VMX transient structure.
597 *
598 * @returns VBox status code.
599 * @param pVmxTransient Pointer to the VMX transient structure.
600 *
601 * @remarks No-long-jump zone!!!
602 */
603DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
604{
605 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
606 AssertRCReturn(rc, rc);
607 return VINF_SUCCESS;
608}
609
610
611/**
612 * Reads the VM-exit interruption-information field from the VMCS into the VMX
613 * transient structure.
614 *
615 * @returns VBox status code.
616 * @param pVmxTransient Pointer to the VMX transient structure.
617 */
618DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
619{
620 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
621 {
622 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
623 AssertRCReturn(rc, rc);
624 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
625 }
626 return VINF_SUCCESS;
627}
628
629
630/**
631 * Reads the VM-exit interruption error code from the VMCS into the VMX
632 * transient structure.
633 *
634 * @returns VBox status code.
635 * @param pVmxTransient Pointer to the VMX transient structure.
636 */
637DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
638{
639 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
640 {
641 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
642 AssertRCReturn(rc, rc);
643 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
644 }
645 return VINF_SUCCESS;
646}
647
648
649/**
650 * Reads the VM-exit instruction length field from the VMCS into the VMX
651 * transient structure.
652 *
653 * @returns VBox status code.
654 * @param pVCpu Pointer to the VMCPU.
655 * @param pVmxTransient Pointer to the VMX transient structure.
656 */
657DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
658{
659 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
660 {
661 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
662 AssertRCReturn(rc, rc);
663 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
664 }
665 return VINF_SUCCESS;
666}
667
668
669/**
670 * Reads the VM-exit instruction-information field from the VMCS into
671 * the VMX transient structure.
672 *
673 * @returns VBox status code.
674 * @param pVmxTransient Pointer to the VMX transient structure.
675 */
676DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
677{
678 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
679 {
680 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
681 AssertRCReturn(rc, rc);
682 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
683 }
684 return VINF_SUCCESS;
685}
686
687
688/**
689 * Reads the exit code qualification from the VMCS into the VMX transient
690 * structure.
691 *
692 * @returns VBox status code.
693 * @param pVCpu Pointer to the VMCPU (required for the VMCS cache
694 * case).
695 * @param pVmxTransient Pointer to the VMX transient structure.
696 */
697DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
698{
699 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
700 {
701 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
702 AssertRCReturn(rc, rc);
703 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
704 }
705 return VINF_SUCCESS;
706}
707
708
709/**
710 * Reads the IDT-vectoring information field from the VMCS into the VMX
711 * transient structure.
712 *
713 * @returns VBox status code.
714 * @param pVmxTransient Pointer to the VMX transient structure.
715 *
716 * @remarks No-long-jump zone!!!
717 */
718DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
719{
720 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
721 {
722 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
723 AssertRCReturn(rc, rc);
724 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
725 }
726 return VINF_SUCCESS;
727}
728
729
730/**
731 * Reads the IDT-vectoring error code from the VMCS into the VMX
732 * transient structure.
733 *
734 * @returns VBox status code.
735 * @param pVmxTransient Pointer to the VMX transient structure.
736 */
737DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
738{
739 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
740 {
741 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
742 AssertRCReturn(rc, rc);
743 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
744 }
745 return VINF_SUCCESS;
746}
747
748
749/**
750 * Enters VMX root mode operation on the current CPU.
751 *
752 * @returns VBox status code.
753 * @param pVM Pointer to the VM (optional, can be NULL, after
754 * a resume).
755 * @param HCPhysCpuPage Physical address of the VMXON region.
756 * @param pvCpuPage Pointer to the VMXON region.
757 */
758static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
759{
760 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
761 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
762 Assert(pvCpuPage);
763 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
764
765 if (pVM)
766 {
767 /* Write the VMCS revision dword to the VMXON region. */
768 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
769 }
770
771 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
772 RTCCUINTREG uEflags = ASMIntDisableFlags();
773
774 /* Enable the VMX bit in CR4 if necessary. */
775 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
776
777 /* Enter VMX root mode. */
778 int rc = VMXEnable(HCPhysCpuPage);
779 if ( RT_FAILURE(rc)
780 && !(uOldCr4 & X86_CR4_VMXE))
781 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
782
783 /* Restore interrupts. */
784 ASMSetFlags(uEflags);
785 return rc;
786}
787
788
789/**
790 * Exits VMX root mode operation on the current CPU.
791 *
792 * @returns VBox status code.
793 */
794static int hmR0VmxLeaveRootMode(void)
795{
796 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
797
798 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
799 RTCCUINTREG uEflags = ASMIntDisableFlags();
800
801 /* If we're for some reason not in VMX root mode, then don't leave it. */
802 RTCCUINTREG uHostCR4 = ASMGetCR4();
803
804 int rc;
805 if (uHostCR4 & X86_CR4_VMXE)
806 {
807 /* Exit VMX root mode and clear the VMX bit in CR4. */
808 VMXDisable();
809 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
810 rc = VINF_SUCCESS;
811 }
812 else
813 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
814
815 /* Restore interrupts. */
816 ASMSetFlags(uEflags);
817 return rc;
818}
819
820
821/**
822 * Allocates and maps one physically contiguous page. The allocated page is
823 * zero'd out. (Used by various VT-x structures).
824 *
825 * @returns IPRT status code.
826 * @param pMemObj Pointer to the ring-0 memory object.
827 * @param ppVirt Where to store the virtual address of the
828 * allocation.
829 * @param pPhys Where to store the physical address of the
830 * allocation.
831 */
832DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
833{
834 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
835 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
836 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
837
838 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
839 if (RT_FAILURE(rc))
840 return rc;
841 *ppVirt = RTR0MemObjAddress(*pMemObj);
842 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
843 ASMMemZero32(*ppVirt, PAGE_SIZE);
844 return VINF_SUCCESS;
845}
846
847
848/**
849 * Frees and unmaps an allocated physical page.
850 *
851 * @param pMemObj Pointer to the ring-0 memory object.
852 * @param ppVirt Where to re-initialize the virtual address of
853 * allocation as 0.
854 * @param pHCPhys Where to re-initialize the physical address of the
855 * allocation as 0.
856 */
857DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
858{
859 AssertPtr(pMemObj);
860 AssertPtr(ppVirt);
861 AssertPtr(pHCPhys);
862 if (*pMemObj != NIL_RTR0MEMOBJ)
863 {
864 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
865 AssertRC(rc);
866 *pMemObj = NIL_RTR0MEMOBJ;
867 *ppVirt = 0;
868 *pHCPhys = 0;
869 }
870}
871
872
873/**
874 * Worker function to free VT-x related structures.
875 *
876 * @returns IPRT status code.
877 * @param pVM Pointer to the VM.
878 */
879static void hmR0VmxStructsFree(PVM pVM)
880{
881 for (VMCPUID i = 0; i < pVM->cCpus; i++)
882 {
883 PVMCPU pVCpu = &pVM->aCpus[i];
884 AssertPtr(pVCpu);
885
886 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
887 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
888
889 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
890 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
891
892 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
893 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
894 }
895
896 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
897#ifdef VBOX_WITH_CRASHDUMP_MAGIC
898 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
899#endif
900}
901
902
903/**
904 * Worker function to allocate VT-x related VM structures.
905 *
906 * @returns IPRT status code.
907 * @param pVM Pointer to the VM.
908 */
909static int hmR0VmxStructsAlloc(PVM pVM)
910{
911 /*
912 * Initialize members up-front so we can cleanup properly on allocation failure.
913 */
914#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
915 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
916 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
917 pVM->hm.s.vmx.HCPhys##a_Name = 0;
918
919#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
920 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
921 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
922 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
923
924#ifdef VBOX_WITH_CRASHDUMP_MAGIC
925 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
926#endif
927 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
928
929 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
930 for (VMCPUID i = 0; i < pVM->cCpus; i++)
931 {
932 PVMCPU pVCpu = &pVM->aCpus[i];
933 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
934 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
935 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
936 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
937 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
938 }
939#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
940#undef VMXLOCAL_INIT_VM_MEMOBJ
941
942 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
943 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
944 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
945 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
946
947 /*
948 * Allocate all the VT-x structures.
949 */
950 int rc = VINF_SUCCESS;
951#ifdef VBOX_WITH_CRASHDUMP_MAGIC
952 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
953 if (RT_FAILURE(rc))
954 goto cleanup;
955 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
956 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
957#endif
958
959 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
960 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
961 {
962 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
963 &pVM->hm.s.vmx.HCPhysApicAccess);
964 if (RT_FAILURE(rc))
965 goto cleanup;
966 }
967
968 /*
969 * Initialize per-VCPU VT-x structures.
970 */
971 for (VMCPUID i = 0; i < pVM->cCpus; i++)
972 {
973 PVMCPU pVCpu = &pVM->aCpus[i];
974 AssertPtr(pVCpu);
975
976 /* Allocate the VM control structure (VMCS). */
977 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
978 if (RT_FAILURE(rc))
979 goto cleanup;
980
981 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
982 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
983 {
984 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
985 &pVCpu->hm.s.vmx.HCPhysVirtApic);
986 if (RT_FAILURE(rc))
987 goto cleanup;
988 }
989
990 /*
991 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
992 * transparent accesses of specific MSRs.
993 *
994 * If the condition for enabling MSR bitmaps changes here, don't forget to
995 * update HMIsMsrBitmapsAvailable().
996 */
997 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
998 {
999 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1000 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1001 if (RT_FAILURE(rc))
1002 goto cleanup;
1003 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1004 }
1005
1006 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1007 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1008 if (RT_FAILURE(rc))
1009 goto cleanup;
1010
1011 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1012 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1013 if (RT_FAILURE(rc))
1014 goto cleanup;
1015 }
1016
1017 return VINF_SUCCESS;
1018
1019cleanup:
1020 hmR0VmxStructsFree(pVM);
1021 return rc;
1022}
1023
1024
1025/**
1026 * Does global VT-x initialization (called during module initialization).
1027 *
1028 * @returns VBox status code.
1029 */
1030VMMR0DECL(int) VMXR0GlobalInit(void)
1031{
1032#ifdef HMVMX_USE_FUNCTION_TABLE
1033 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1034# ifdef VBOX_STRICT
1035 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1036 Assert(g_apfnVMExitHandlers[i]);
1037# endif
1038#endif
1039 return VINF_SUCCESS;
1040}
1041
1042
1043/**
1044 * Does global VT-x termination (called during module termination).
1045 */
1046VMMR0DECL(void) VMXR0GlobalTerm()
1047{
1048 /* Nothing to do currently. */
1049}
1050
1051
1052/**
1053 * Sets up and activates VT-x on the current CPU.
1054 *
1055 * @returns VBox status code.
1056 * @param pCpu Pointer to the global CPU info struct.
1057 * @param pVM Pointer to the VM (can be NULL after a host resume
1058 * operation).
1059 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1060 * fEnabledByHost is true).
1061 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1062 * @a fEnabledByHost is true).
1063 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1064 * enable VT-x on the host.
1065 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1066 */
1067VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1068 void *pvMsrs)
1069{
1070 Assert(pCpu);
1071 Assert(pvMsrs);
1072 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1073
1074 /* Enable VT-x if it's not already enabled by the host. */
1075 if (!fEnabledByHost)
1076 {
1077 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1078 if (RT_FAILURE(rc))
1079 return rc;
1080 }
1081
1082 /*
1083 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1084 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1085 */
1086 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1087 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1088 {
1089 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1090 pCpu->fFlushAsidBeforeUse = false;
1091 }
1092 else
1093 pCpu->fFlushAsidBeforeUse = true;
1094
1095 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1096 ++pCpu->cTlbFlushes;
1097
1098 return VINF_SUCCESS;
1099}
1100
1101
1102/**
1103 * Deactivates VT-x on the current CPU.
1104 *
1105 * @returns VBox status code.
1106 * @param pCpu Pointer to the global CPU info struct.
1107 * @param pvCpuPage Pointer to the VMXON region.
1108 * @param HCPhysCpuPage Physical address of the VMXON region.
1109 *
1110 * @remarks This function should never be called when SUPR0EnableVTx() or
1111 * similar was used to enable VT-x on the host.
1112 */
1113VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1114{
1115 NOREF(pCpu);
1116 NOREF(pvCpuPage);
1117 NOREF(HCPhysCpuPage);
1118
1119 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1120 return hmR0VmxLeaveRootMode();
1121}
1122
1123
1124/**
1125 * Sets the permission bits for the specified MSR in the MSR bitmap.
1126 *
1127 * @param pVCpu Pointer to the VMCPU.
1128 * @param uMSR The MSR value.
1129 * @param enmRead Whether reading this MSR causes a VM-exit.
1130 * @param enmWrite Whether writing this MSR causes a VM-exit.
1131 */
1132static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1133{
1134 int32_t iBit;
1135 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1136
1137 /*
1138 * Layout:
1139 * 0x000 - 0x3ff - Low MSR read bits
1140 * 0x400 - 0x7ff - High MSR read bits
1141 * 0x800 - 0xbff - Low MSR write bits
1142 * 0xc00 - 0xfff - High MSR write bits
1143 */
1144 if (uMsr <= 0x00001FFF)
1145 iBit = uMsr;
1146 else if ( uMsr >= 0xC0000000
1147 && uMsr <= 0xC0001FFF)
1148 {
1149 iBit = (uMsr - 0xC0000000);
1150 pbMsrBitmap += 0x400;
1151 }
1152 else
1153 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1154
1155 Assert(iBit <= 0x1fff);
1156 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1157 ASMBitSet(pbMsrBitmap, iBit);
1158 else
1159 ASMBitClear(pbMsrBitmap, iBit);
1160
1161 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1162 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1163 else
1164 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1165}
1166
1167
1168#ifdef VBOX_STRICT
1169/**
1170 * Gets the permission bits for the specified MSR in the MSR bitmap.
1171 *
1172 * @returns VBox status code.
1173 * @retval VINF_SUCCESS if the specified MSR is found.
1174 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1175 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1176 *
1177 * @param pVCpu Pointer to the VMCPU.
1178 * @param uMsr The MSR.
1179 * @param penmRead Where to store the read permissions.
1180 * @param penmWrite Where to store the write permissions.
1181 */
1182static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1183{
1184 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1185 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1186 int32_t iBit;
1187 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1188
1189 /* See hmR0VmxSetMsrPermission() for the layout. */
1190 if (uMsr <= 0x00001FFF)
1191 iBit = uMsr;
1192 else if ( uMsr >= 0xC0000000
1193 && uMsr <= 0xC0001FFF)
1194 {
1195 iBit = (uMsr - 0xC0000000);
1196 pbMsrBitmap += 0x400;
1197 }
1198 else
1199 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1200
1201 Assert(iBit <= 0x1fff);
1202 if (ASMBitTest(pbMsrBitmap, iBit))
1203 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1204 else
1205 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1206
1207 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1208 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1209 else
1210 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1211 return VINF_SUCCESS;
1212}
1213#endif /* VBOX_STRICT */
1214
1215
1216/**
1217 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1218 * area.
1219 *
1220 * @returns VBox status code.
1221 * @param pVCpu Pointer to the VMCPU.
1222 * @param cMsrs The number of MSRs.
1223 */
1224DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1225{
1226 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1227 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1228 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1229 {
1230 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1231 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1232 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1233 }
1234
1235 /* Update number of guest MSRs to load/store across the world-switch. */
1236 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1237 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1238
1239 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1240 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1241
1242 /* Update the VCPU's copy of the MSR count. */
1243 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1244
1245 return VINF_SUCCESS;
1246}
1247
1248
1249/**
1250 * Adds a new (or updates the value of an existing) guest/host MSR
1251 * pair to be swapped during the world-switch as part of the
1252 * auto-load/store MSR area in the VMCS.
1253 *
1254 * @returns true if the MSR was added -and- its value was updated, false
1255 * otherwise.
1256 * @param pVCpu Pointer to the VMCPU.
1257 * @param uMsr The MSR.
1258 * @param uGuestMsr Value of the guest MSR.
1259 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1260 * necessary.
1261 */
1262static bool hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1263{
1264 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1265 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1266 uint32_t i;
1267 for (i = 0; i < cMsrs; i++)
1268 {
1269 if (pGuestMsr->u32Msr == uMsr)
1270 break;
1271 pGuestMsr++;
1272 }
1273
1274 bool fAdded = false;
1275 if (i == cMsrs)
1276 {
1277 ++cMsrs;
1278 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1279 AssertRC(rc);
1280
1281 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1282 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1283 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1284
1285 fAdded = true;
1286 }
1287
1288 /* Update the MSR values in the auto-load/store MSR area. */
1289 pGuestMsr->u32Msr = uMsr;
1290 pGuestMsr->u64Value = uGuestMsrValue;
1291
1292 /* Create/update the MSR slot in the host MSR area. */
1293 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1294 pHostMsr += i;
1295 pHostMsr->u32Msr = uMsr;
1296
1297 /*
1298 * Update the host MSR only when requested by the caller AND when we're
1299 * adding it to the auto-load/store area. Otherwise, it would have been
1300 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1301 */
1302 bool fUpdatedMsrValue = false;
1303 if ( fAdded
1304 && fUpdateHostMsr)
1305 {
1306 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1307 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1308 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1309 fUpdatedMsrValue = true;
1310 }
1311
1312 return fUpdatedMsrValue;
1313}
1314
1315
1316/**
1317 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1318 * auto-load/store MSR area in the VMCS.
1319 *
1320 * @returns VBox status code.
1321 * @param pVCpu Pointer to the VMCPU.
1322 * @param uMsr The MSR.
1323 */
1324static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1325{
1326 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1327 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1328 for (uint32_t i = 0; i < cMsrs; i++)
1329 {
1330 /* Find the MSR. */
1331 if (pGuestMsr->u32Msr == uMsr)
1332 {
1333 /* If it's the last MSR, simply reduce the count. */
1334 if (i == cMsrs - 1)
1335 {
1336 --cMsrs;
1337 break;
1338 }
1339
1340 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1341 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1342 pLastGuestMsr += cMsrs - 1;
1343 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1344 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1345
1346 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1347 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1348 pLastHostMsr += cMsrs - 1;
1349 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1350 pHostMsr->u64Value = pLastHostMsr->u64Value;
1351 --cMsrs;
1352 break;
1353 }
1354 pGuestMsr++;
1355 }
1356
1357 /* Update the VMCS if the count changed (meaning the MSR was found). */
1358 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1359 {
1360 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1361 AssertRCReturn(rc, rc);
1362
1363 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1364 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1365 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1366
1367 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1368 return VINF_SUCCESS;
1369 }
1370
1371 return VERR_NOT_FOUND;
1372}
1373
1374
1375/**
1376 * Checks if the specified guest MSR is part of the auto-load/store area in
1377 * the VMCS.
1378 *
1379 * @returns true if found, false otherwise.
1380 * @param pVCpu Pointer to the VMCPU.
1381 * @param uMsr The MSR to find.
1382 */
1383static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1384{
1385 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1386 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1387
1388 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1389 {
1390 if (pGuestMsr->u32Msr == uMsr)
1391 return true;
1392 }
1393 return false;
1394}
1395
1396
1397/**
1398 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1399 *
1400 * @param pVCpu Pointer to the VMCPU.
1401 *
1402 * @remarks No-long-jump zone!!!
1403 */
1404static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1405{
1406 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1407 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1408 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1409 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1410
1411 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1412 {
1413 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1414
1415 /*
1416 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1417 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1418 */
1419 if (pHostMsr->u32Msr == MSR_K6_EFER)
1420 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1421 else
1422 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1423 }
1424
1425 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1426}
1427
1428
1429#if HC_ARCH_BITS == 64
1430/**
1431 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1432 * perform lazy restoration of the host MSRs while leaving VT-x.
1433 *
1434 * @param pVCpu Pointer to the VMCPU.
1435 *
1436 * @remarks No-long-jump zone!!!
1437 */
1438static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1439{
1440 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1441
1442 /*
1443 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1444 */
1445 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1446 {
1447 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1448 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1449 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1450 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1451 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1452 }
1453}
1454
1455
1456/**
1457 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1458 * lazily while leaving VT-x.
1459 *
1460 * @returns true if it does, false otherwise.
1461 * @param pVCpu Pointer to the VMCPU.
1462 * @param uMsr The MSR to check.
1463 */
1464static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1465{
1466 NOREF(pVCpu);
1467 switch (uMsr)
1468 {
1469 case MSR_K8_LSTAR:
1470 case MSR_K6_STAR:
1471 case MSR_K8_SF_MASK:
1472 case MSR_K8_KERNEL_GS_BASE:
1473 return true;
1474 }
1475 return false;
1476}
1477
1478
1479/**
1480 * Saves a set of guest MSRs back into the guest-CPU context.
1481 *
1482 * @param pVCpu Pointer to the VMCPU.
1483 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1484 * out-of-sync. Make sure to update the required fields
1485 * before using them.
1486 *
1487 * @remarks No-long-jump zone!!!
1488 */
1489static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1490{
1491 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1492 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1493
1494 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1495 {
1496 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1497 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1498 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1499 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1500 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1501 }
1502}
1503
1504
1505/**
1506 * Loads a set of guests MSRs to allow read/passthru to the guest.
1507 *
1508 * The name of this function is slightly confusing. This function does NOT
1509 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1510 * common prefix for functions dealing with "lazy restoration" of the shared
1511 * MSRs.
1512 *
1513 * @param pVCpu Pointer to the VMCPU.
1514 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1515 * out-of-sync. Make sure to update the required fields
1516 * before using them.
1517 *
1518 * @remarks No-long-jump zone!!!
1519 */
1520static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1521{
1522 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1523 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1524
1525#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1526 do { \
1527 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1528 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1529 else \
1530 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1531 } while (0)
1532
1533 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1534 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1535 {
1536 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1537 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1538 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1539 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1540 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1541 }
1542 else
1543 {
1544 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1545 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1546 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1547 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1548 }
1549
1550#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1551}
1552
1553
1554/**
1555 * Performs lazy restoration of the set of host MSRs if they were previously
1556 * loaded with guest MSR values.
1557 *
1558 * @param pVCpu Pointer to the VMCPU.
1559 *
1560 * @remarks No-long-jump zone!!!
1561 * @remarks The guest MSRs should have been saved back into the guest-CPU
1562 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1563 */
1564static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1565{
1566 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1567 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1568
1569 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1570 {
1571 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1572 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1573 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1574 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1575 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1576 }
1577 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1578}
1579#endif /* HC_ARCH_BITS == 64 */
1580
1581
1582/**
1583 * Verifies that our cached values of the VMCS controls are all
1584 * consistent with what's actually present in the VMCS.
1585 *
1586 * @returns VBox status code.
1587 * @param pVCpu Pointer to the VMCPU.
1588 */
1589static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1590{
1591 uint32_t u32Val;
1592 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1593 AssertRCReturn(rc, rc);
1594 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1595 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1596
1597 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1598 AssertRCReturn(rc, rc);
1599 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1600 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1601
1602 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1603 AssertRCReturn(rc, rc);
1604 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1605 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1606
1607 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1608 AssertRCReturn(rc, rc);
1609 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1610 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1611
1612 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1613 {
1614 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1615 AssertRCReturn(rc, rc);
1616 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1617 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1618 }
1619
1620 return VINF_SUCCESS;
1621}
1622
1623
1624#ifdef VBOX_STRICT
1625/**
1626 * Verifies that our cached host EFER value has not changed
1627 * since we cached it.
1628 *
1629 * @param pVCpu Pointer to the VMCPU.
1630 */
1631static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1632{
1633 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1634
1635 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1636 {
1637 uint64_t u64Val;
1638 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, &u64Val);
1639 AssertRC(rc);
1640
1641 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1642 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1643 }
1644}
1645
1646
1647/**
1648 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1649 * VMCS are correct.
1650 *
1651 * @param pVCpu Pointer to the VMCPU.
1652 */
1653static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1654{
1655 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1656
1657 /* Verify MSR counts in the VMCS are what we think it should be. */
1658 uint32_t cMsrs;
1659 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1660 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1661
1662 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1663 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1664
1665 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1666 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1667
1668 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1669 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1670 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1671 {
1672 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1673 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1674 pGuestMsr->u32Msr, cMsrs));
1675
1676 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1677 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1678 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1679
1680 /* Verify that the permissions are as expected in the MSR bitmap. */
1681 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1682 {
1683 VMXMSREXITREAD enmRead;
1684 VMXMSREXITWRITE enmWrite;
1685 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1686 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1687 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1688 {
1689 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1690 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1691 }
1692 else
1693 {
1694 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1695 pGuestMsr->u32Msr, cMsrs));
1696 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1697 pGuestMsr->u32Msr, cMsrs));
1698 }
1699 }
1700 }
1701}
1702#endif /* VBOX_STRICT */
1703
1704
1705/**
1706 * Flushes the TLB using EPT.
1707 *
1708 * @returns VBox status code.
1709 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1710 * enmFlush).
1711 * @param enmFlush Type of flush.
1712 *
1713 * @remarks Caller is responsible for making sure this function is called only
1714 * when NestedPaging is supported and providing @a enmFlush that is
1715 * supported by the CPU.
1716 * @remarks Can be called with interrupts disabled.
1717 */
1718static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1719{
1720 uint64_t au64Descriptor[2];
1721 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1722 au64Descriptor[0] = 0;
1723 else
1724 {
1725 Assert(pVCpu);
1726 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1727 }
1728 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1729
1730 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1731 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1732 rc));
1733 if ( RT_SUCCESS(rc)
1734 && pVCpu)
1735 {
1736 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1737 }
1738}
1739
1740
1741/**
1742 * Flushes the TLB using VPID.
1743 *
1744 * @returns VBox status code.
1745 * @param pVM Pointer to the VM.
1746 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1747 * enmFlush).
1748 * @param enmFlush Type of flush.
1749 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1750 * on @a enmFlush).
1751 *
1752 * @remarks Can be called with interrupts disabled.
1753 */
1754static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1755{
1756 NOREF(pVM);
1757 AssertPtr(pVM);
1758 Assert(pVM->hm.s.vmx.fVpid);
1759
1760 uint64_t au64Descriptor[2];
1761 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1762 {
1763 au64Descriptor[0] = 0;
1764 au64Descriptor[1] = 0;
1765 }
1766 else
1767 {
1768 AssertPtr(pVCpu);
1769 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1770 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1771 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1772 au64Descriptor[1] = GCPtr;
1773 }
1774
1775 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1776 AssertMsg(rc == VINF_SUCCESS,
1777 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1778 if ( RT_SUCCESS(rc)
1779 && pVCpu)
1780 {
1781 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1782 }
1783}
1784
1785
1786/**
1787 * Invalidates a guest page by guest virtual address. Only relevant for
1788 * EPT/VPID, otherwise there is nothing really to invalidate.
1789 *
1790 * @returns VBox status code.
1791 * @param pVM Pointer to the VM.
1792 * @param pVCpu Pointer to the VMCPU.
1793 * @param GCVirt Guest virtual address of the page to invalidate.
1794 */
1795VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1796{
1797 AssertPtr(pVM);
1798 AssertPtr(pVCpu);
1799 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1800
1801 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1802 if (!fFlushPending)
1803 {
1804 /*
1805 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1806 * See @bugref{6043} and @bugref{6177}.
1807 *
1808 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1809 * function maybe called in a loop with individual addresses.
1810 */
1811 if (pVM->hm.s.vmx.fVpid)
1812 {
1813 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1814 {
1815 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1816 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1817 }
1818 else
1819 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1820 }
1821 else if (pVM->hm.s.fNestedPaging)
1822 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1823 }
1824
1825 return VINF_SUCCESS;
1826}
1827
1828
1829/**
1830 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1831 * otherwise there is nothing really to invalidate.
1832 *
1833 * @returns VBox status code.
1834 * @param pVM Pointer to the VM.
1835 * @param pVCpu Pointer to the VMCPU.
1836 * @param GCPhys Guest physical address of the page to invalidate.
1837 */
1838VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1839{
1840 NOREF(pVM); NOREF(GCPhys);
1841 LogFlowFunc(("%RGp\n", GCPhys));
1842
1843 /*
1844 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1845 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1846 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1847 */
1848 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1849 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1850 return VINF_SUCCESS;
1851}
1852
1853
1854/**
1855 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1856 * case where neither EPT nor VPID is supported by the CPU.
1857 *
1858 * @param pVM Pointer to the VM.
1859 * @param pVCpu Pointer to the VMCPU.
1860 * @param pCpu Pointer to the global HM struct.
1861 *
1862 * @remarks Called with interrupts disabled.
1863 */
1864static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1865{
1866 AssertPtr(pVCpu);
1867 AssertPtr(pCpu);
1868 NOREF(pVM);
1869
1870 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1871
1872 /** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1873#if 0
1874 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1875 pVCpu->hm.s.TlbShootdown.cPages = 0;
1876#endif
1877
1878 Assert(pCpu->idCpu != NIL_RTCPUID);
1879 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1880 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1881 pVCpu->hm.s.fForceTLBFlush = false;
1882 return;
1883}
1884
1885
1886/**
1887 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1888 *
1889 * @param pVM Pointer to the VM.
1890 * @param pVCpu Pointer to the VMCPU.
1891 * @param pCpu Pointer to the global HM CPU struct.
1892 * @remarks All references to "ASID" in this function pertains to "VPID" in
1893 * Intel's nomenclature. The reason is, to avoid confusion in compare
1894 * statements since the host-CPU copies are named "ASID".
1895 *
1896 * @remarks Called with interrupts disabled.
1897 */
1898static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1899{
1900#ifdef VBOX_WITH_STATISTICS
1901 bool fTlbFlushed = false;
1902# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1903# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1904 if (!fTlbFlushed) \
1905 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1906 } while (0)
1907#else
1908# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1909# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1910#endif
1911
1912 AssertPtr(pVM);
1913 AssertPtr(pCpu);
1914 AssertPtr(pVCpu);
1915 Assert(pCpu->idCpu != NIL_RTCPUID);
1916
1917 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1918 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1919 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1920
1921 /*
1922 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1923 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1924 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1925 */
1926 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1927 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1928 {
1929 ++pCpu->uCurrentAsid;
1930 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1931 {
1932 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1933 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1934 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1935 }
1936
1937 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1938 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1939 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1940
1941 /*
1942 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1943 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1944 */
1945 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1946 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1947 HMVMX_SET_TAGGED_TLB_FLUSHED();
1948 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1949 }
1950
1951 /* Check for explicit TLB shootdowns. */
1952 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1953 {
1954 /*
1955 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1956 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1957 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1958 * but not guest-physical mappings.
1959 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1960 */
1961 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1962 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1963 HMVMX_SET_TAGGED_TLB_FLUSHED();
1964 }
1965
1966 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1967 * where it is commented out. Support individual entry flushing
1968 * someday. */
1969#if 0
1970 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1971 {
1972 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1973
1974 /*
1975 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1976 * as supported by the CPU.
1977 */
1978 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1979 {
1980 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1981 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1982 }
1983 else
1984 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1985
1986 HMVMX_SET_TAGGED_TLB_FLUSHED();
1987 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1988 pVCpu->hm.s.TlbShootdown.cPages = 0;
1989 }
1990#endif
1991
1992 pVCpu->hm.s.fForceTLBFlush = false;
1993
1994 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1995
1996 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1997 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1998 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1999 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2000 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2001 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2002 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2003 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2004 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2005
2006 /* Update VMCS with the VPID. */
2007 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2008 AssertRC(rc);
2009
2010#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2011}
2012
2013
2014/**
2015 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2016 *
2017 * @returns VBox status code.
2018 * @param pVM Pointer to the VM.
2019 * @param pVCpu Pointer to the VMCPU.
2020 * @param pCpu Pointer to the global HM CPU struct.
2021 *
2022 * @remarks Called with interrupts disabled.
2023 */
2024static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2025{
2026 AssertPtr(pVM);
2027 AssertPtr(pVCpu);
2028 AssertPtr(pCpu);
2029 Assert(pCpu->idCpu != NIL_RTCPUID);
2030 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2031 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2032
2033 /*
2034 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2035 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2036 */
2037 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2038 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2039 {
2040 pVCpu->hm.s.fForceTLBFlush = true;
2041 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2042 }
2043
2044 /* Check for explicit TLB shootdown flushes. */
2045 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2046 {
2047 pVCpu->hm.s.fForceTLBFlush = true;
2048 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2049 }
2050
2051 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2052 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2053
2054 if (pVCpu->hm.s.fForceTLBFlush)
2055 {
2056 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2057 pVCpu->hm.s.fForceTLBFlush = false;
2058 }
2059 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2060 * where it is commented out. Support individual entry flushing
2061 * someday. */
2062#if 0
2063 else
2064 {
2065 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2066 {
2067 /* We cannot flush individual entries without VPID support. Flush using EPT. */
2068 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
2069 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2070 }
2071 else
2072 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2073
2074 pVCpu->hm.s.TlbShootdown.cPages = 0;
2075 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2076 }
2077#endif
2078}
2079
2080
2081/**
2082 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2083 *
2084 * @returns VBox status code.
2085 * @param pVM Pointer to the VM.
2086 * @param pVCpu Pointer to the VMCPU.
2087 * @param pCpu Pointer to the global HM CPU struct.
2088 *
2089 * @remarks Called with interrupts disabled.
2090 */
2091static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2092{
2093 AssertPtr(pVM);
2094 AssertPtr(pVCpu);
2095 AssertPtr(pCpu);
2096 Assert(pCpu->idCpu != NIL_RTCPUID);
2097 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2098 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2099
2100 /*
2101 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2102 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2103 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2104 */
2105 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2106 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2107 {
2108 pVCpu->hm.s.fForceTLBFlush = true;
2109 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2110 }
2111
2112 /* Check for explicit TLB shootdown flushes. */
2113 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2114 {
2115 /*
2116 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2117 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2118 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2119 */
2120 pVCpu->hm.s.fForceTLBFlush = true;
2121 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2122 }
2123
2124 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2125 if (pVCpu->hm.s.fForceTLBFlush)
2126 {
2127 ++pCpu->uCurrentAsid;
2128 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2129 {
2130 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2131 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2132 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2133 }
2134
2135 pVCpu->hm.s.fForceTLBFlush = false;
2136 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2137 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2138 if (pCpu->fFlushAsidBeforeUse)
2139 {
2140 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2141 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2142 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2143 {
2144 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2145 pCpu->fFlushAsidBeforeUse = false;
2146 }
2147 else
2148 {
2149 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2150 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2151 }
2152 }
2153 }
2154 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2155 * where it is commented out. Support individual entry flushing
2156 * someday. */
2157#if 0
2158 else
2159 {
2160 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2161 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2162 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2163 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2164
2165 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2166 {
2167 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2168 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2169 {
2170 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2171 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2172 }
2173 else
2174 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2175
2176 pVCpu->hm.s.TlbShootdown.cPages = 0;
2177 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2178 }
2179 else
2180 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2181 }
2182#endif
2183
2184 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2185 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2186 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2187 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2188 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2189 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2190 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2191
2192 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2193 AssertRC(rc);
2194}
2195
2196
2197/**
2198 * Flushes the guest TLB entry based on CPU capabilities.
2199 *
2200 * @param pVCpu Pointer to the VMCPU.
2201 * @param pCpu Pointer to the global HM CPU struct.
2202 */
2203DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2204{
2205#ifdef HMVMX_ALWAYS_FLUSH_TLB
2206 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2207#endif
2208 PVM pVM = pVCpu->CTX_SUFF(pVM);
2209 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2210 {
2211 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2212 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2213 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2214 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2215 default:
2216 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2217 break;
2218 }
2219
2220 /* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2221 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2222
2223 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2224}
2225
2226
2227/**
2228 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2229 * TLB entries from the host TLB before VM-entry.
2230 *
2231 * @returns VBox status code.
2232 * @param pVM Pointer to the VM.
2233 */
2234static int hmR0VmxSetupTaggedTlb(PVM pVM)
2235{
2236 /*
2237 * Determine optimal flush type for Nested Paging.
2238 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2239 * guest execution (see hmR3InitFinalizeR0()).
2240 */
2241 if (pVM->hm.s.fNestedPaging)
2242 {
2243 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2244 {
2245 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2246 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2247 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2248 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2249 else
2250 {
2251 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2252 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2253 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2254 }
2255
2256 /* Make sure the write-back cacheable memory type for EPT is supported. */
2257 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2258 {
2259 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2260 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2261 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2262 }
2263 }
2264 else
2265 {
2266 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2267 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2268 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2269 }
2270 }
2271
2272 /*
2273 * Determine optimal flush type for VPID.
2274 */
2275 if (pVM->hm.s.vmx.fVpid)
2276 {
2277 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2278 {
2279 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2280 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2281 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2282 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2283 else
2284 {
2285 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2286 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2287 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2288 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2289 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2290 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2291 pVM->hm.s.vmx.fVpid = false;
2292 }
2293 }
2294 else
2295 {
2296 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2297 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2298 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2299 pVM->hm.s.vmx.fVpid = false;
2300 }
2301 }
2302
2303 /*
2304 * Setup the handler for flushing tagged-TLBs.
2305 */
2306 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2307 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2308 else if (pVM->hm.s.fNestedPaging)
2309 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2310 else if (pVM->hm.s.vmx.fVpid)
2311 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2312 else
2313 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2314 return VINF_SUCCESS;
2315}
2316
2317
2318/**
2319 * Sets up pin-based VM-execution controls in the VMCS.
2320 *
2321 * @returns VBox status code.
2322 * @param pVM Pointer to the VM.
2323 * @param pVCpu Pointer to the VMCPU.
2324 */
2325static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2326{
2327 AssertPtr(pVM);
2328 AssertPtr(pVCpu);
2329
2330 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2331 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2332
2333 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2334 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2335
2336 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2337 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2338
2339 /* Enable the VMX preemption timer. */
2340 if (pVM->hm.s.vmx.fUsePreemptTimer)
2341 {
2342 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2343 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2344 }
2345
2346 if ((val & zap) != val)
2347 {
2348 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2349 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2350 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2351 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2352 }
2353
2354 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2355 AssertRCReturn(rc, rc);
2356
2357 pVCpu->hm.s.vmx.u32PinCtls = val;
2358 return rc;
2359}
2360
2361
2362/**
2363 * Sets up processor-based VM-execution controls in the VMCS.
2364 *
2365 * @returns VBox status code.
2366 * @param pVM Pointer to the VM.
2367 * @param pVMCPU Pointer to the VMCPU.
2368 */
2369static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2370{
2371 AssertPtr(pVM);
2372 AssertPtr(pVCpu);
2373
2374 int rc = VERR_INTERNAL_ERROR_5;
2375 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2376 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2377
2378 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2379 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2380 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2381 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2382 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2383 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2384 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2385
2386 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2387 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2388 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2389 {
2390 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2391 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2392 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2393 }
2394
2395 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2396 if (!pVM->hm.s.fNestedPaging)
2397 {
2398 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2399 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2400 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2401 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2402 }
2403
2404 /* Use TPR shadowing if supported by the CPU. */
2405 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2406 {
2407 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2408 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2409 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2410 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2411 AssertRCReturn(rc, rc);
2412
2413 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2414 /* CR8 writes cause a VM-exit based on TPR threshold. */
2415 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2416 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2417 }
2418 else
2419 {
2420 /*
2421 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2422 * Set this control only for 64-bit guests.
2423 */
2424 if (pVM->hm.s.fAllow64BitGuests)
2425 {
2426 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2427 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2428 }
2429 }
2430
2431 /* Use MSR-bitmaps if supported by the CPU. */
2432 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2433 {
2434 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2435
2436 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2437 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2438 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2439 AssertRCReturn(rc, rc);
2440
2441 /*
2442 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2443 * automatically using dedicated fields in the VMCS.
2444 */
2445 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2446 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2447 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2448 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2449 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2450
2451#if HC_ARCH_BITS == 64
2452 /*
2453 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2454 */
2455 if (pVM->hm.s.fAllow64BitGuests)
2456 {
2457 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2458 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2459 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2460 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2461 }
2462#endif
2463 }
2464
2465 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2466 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2467 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2468
2469 if ((val & zap) != val)
2470 {
2471 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2472 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2473 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2474 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2475 }
2476
2477 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2478 AssertRCReturn(rc, rc);
2479
2480 pVCpu->hm.s.vmx.u32ProcCtls = val;
2481
2482 /*
2483 * Secondary processor-based VM-execution controls.
2484 */
2485 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2486 {
2487 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2488 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2489
2490 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2491 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2492
2493 if (pVM->hm.s.fNestedPaging)
2494 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2495 else
2496 {
2497 /*
2498 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2499 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2500 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2501 */
2502 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2503 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2504 }
2505
2506 if (pVM->hm.s.vmx.fVpid)
2507 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2508
2509 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2510 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2511
2512 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2513 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2514 * done dynamically. */
2515 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2516 {
2517 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2518 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2519 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2520 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2521 AssertRCReturn(rc, rc);
2522 }
2523
2524 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2525 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2526
2527 if ((val & zap) != val)
2528 {
2529 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2530 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2531 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2532 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2533 }
2534
2535 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2536 AssertRCReturn(rc, rc);
2537
2538 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2539 }
2540 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2541 {
2542 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2543 "available\n"));
2544 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2545 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2546 }
2547
2548 return VINF_SUCCESS;
2549}
2550
2551
2552/**
2553 * Sets up miscellaneous (everything other than Pin & Processor-based
2554 * VM-execution) control fields in the VMCS.
2555 *
2556 * @returns VBox status code.
2557 * @param pVM Pointer to the VM.
2558 * @param pVCpu Pointer to the VMCPU.
2559 */
2560static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2561{
2562 NOREF(pVM);
2563 AssertPtr(pVM);
2564 AssertPtr(pVCpu);
2565
2566 int rc = VERR_GENERAL_FAILURE;
2567
2568 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2569#if 0
2570 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2571 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2572 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2573
2574 /*
2575 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2576 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2577 * We thus use the exception bitmap to control it rather than use both.
2578 */
2579 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2580 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2581
2582 /** @todo Explore possibility of using IO-bitmaps. */
2583 /* All IO & IOIO instructions cause VM-exits. */
2584 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2585 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2586
2587 /* Initialize the MSR-bitmap area. */
2588 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2589 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2590 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2591#endif
2592
2593 /* Setup MSR auto-load/store area. */
2594 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2595 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2596 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2597 AssertRCReturn(rc, rc);
2598 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2599 AssertRCReturn(rc, rc);
2600
2601 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2602 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2603 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2604 AssertRCReturn(rc, rc);
2605
2606 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2607 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2608 AssertRCReturn(rc, rc);
2609
2610 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2611#if 0
2612 /* Setup debug controls */
2613 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2614 AssertRCReturn(rc, rc);
2615 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2616 AssertRCReturn(rc, rc);
2617#endif
2618
2619 return rc;
2620}
2621
2622
2623/**
2624 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2625 *
2626 * @returns VBox status code.
2627 * @param pVM Pointer to the VM.
2628 * @param pVCpu Pointer to the VMCPU.
2629 */
2630static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2631{
2632 AssertPtr(pVM);
2633 AssertPtr(pVCpu);
2634
2635 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2636
2637 uint32_t u32XcptBitmap = 0;
2638
2639 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2640 if (!pVM->hm.s.fNestedPaging)
2641 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2642
2643 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2644 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2645 AssertRCReturn(rc, rc);
2646 return rc;
2647}
2648
2649
2650/**
2651 * Sets up the initial guest-state mask. The guest-state mask is consulted
2652 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2653 * for the nested virtualization case (as it would cause a VM-exit).
2654 *
2655 * @param pVCpu Pointer to the VMCPU.
2656 */
2657static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2658{
2659 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2660 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2661 return VINF_SUCCESS;
2662}
2663
2664
2665/**
2666 * Does per-VM VT-x initialization.
2667 *
2668 * @returns VBox status code.
2669 * @param pVM Pointer to the VM.
2670 */
2671VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2672{
2673 LogFlowFunc(("pVM=%p\n", pVM));
2674
2675 int rc = hmR0VmxStructsAlloc(pVM);
2676 if (RT_FAILURE(rc))
2677 {
2678 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2679 return rc;
2680 }
2681
2682 return VINF_SUCCESS;
2683}
2684
2685
2686/**
2687 * Does per-VM VT-x termination.
2688 *
2689 * @returns VBox status code.
2690 * @param pVM Pointer to the VM.
2691 */
2692VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2693{
2694 LogFlowFunc(("pVM=%p\n", pVM));
2695
2696#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2697 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2698 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2699#endif
2700 hmR0VmxStructsFree(pVM);
2701 return VINF_SUCCESS;
2702}
2703
2704
2705/**
2706 * Sets up the VM for execution under VT-x.
2707 * This function is only called once per-VM during initialization.
2708 *
2709 * @returns VBox status code.
2710 * @param pVM Pointer to the VM.
2711 */
2712VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2713{
2714 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2715 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2716
2717 LogFlowFunc(("pVM=%p\n", pVM));
2718
2719 /*
2720 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2721 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2722 */
2723 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2724 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2725 || !pVM->hm.s.vmx.pRealModeTSS))
2726 {
2727 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2728 return VERR_INTERNAL_ERROR;
2729 }
2730
2731#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2732 /*
2733 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2734 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2735 */
2736 if ( pVM->hm.s.fAllow64BitGuests
2737 && !HMVMX_IS_64BIT_HOST_MODE())
2738 {
2739 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2740 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2741 }
2742#endif
2743
2744 /* Initialize these always, see hmR3InitFinalizeR0().*/
2745 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2746 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2747
2748 /* Setup the tagged-TLB flush handlers. */
2749 int rc = hmR0VmxSetupTaggedTlb(pVM);
2750 if (RT_FAILURE(rc))
2751 {
2752 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2753 return rc;
2754 }
2755
2756 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2757 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2758#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2759 if ( HMVMX_IS_64BIT_HOST_MODE()
2760 && (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2761 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2762 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2763 {
2764 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2765 }
2766#endif
2767
2768 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2769 {
2770 PVMCPU pVCpu = &pVM->aCpus[i];
2771 AssertPtr(pVCpu);
2772 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2773
2774 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2775 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2776
2777 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2778 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2779 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2780
2781 /* Set revision dword at the beginning of the VMCS structure. */
2782 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2783
2784 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2785 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2786 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2787 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2788
2789 /* Load this VMCS as the current VMCS. */
2790 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2791 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2792 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2793
2794 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2795 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2796 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2797
2798 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2799 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2800 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2801
2802 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2803 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2804 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2805
2806 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2807 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2808 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2809
2810 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2811 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2812 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2813
2814#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2815 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2816 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2817 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2818#endif
2819
2820 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2821 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2822 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2823 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2824
2825 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2826
2827 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2828 }
2829
2830 return VINF_SUCCESS;
2831}
2832
2833
2834/**
2835 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2836 * the VMCS.
2837 *
2838 * @returns VBox status code.
2839 * @param pVM Pointer to the VM.
2840 * @param pVCpu Pointer to the VMCPU.
2841 */
2842DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2843{
2844 NOREF(pVM); NOREF(pVCpu);
2845
2846 RTCCUINTREG uReg = ASMGetCR0();
2847 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2848 AssertRCReturn(rc, rc);
2849
2850#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2851 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2852 if (HMVMX_IS_64BIT_HOST_MODE())
2853 {
2854 uint64_t uRegCR3 = HMR0Get64bitCR3();
2855 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2856 }
2857 else
2858#endif
2859 {
2860 uReg = ASMGetCR3();
2861 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2862 }
2863 AssertRCReturn(rc, rc);
2864
2865 uReg = ASMGetCR4();
2866 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2867 AssertRCReturn(rc, rc);
2868 return rc;
2869}
2870
2871
2872#if HC_ARCH_BITS == 64
2873/**
2874 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2875 * requirements. See hmR0VmxSaveHostSegmentRegs().
2876 */
2877# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2878 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2879 { \
2880 bool fValidSelector = true; \
2881 if ((selValue) & X86_SEL_LDT) \
2882 { \
2883 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2884 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2885 } \
2886 if (fValidSelector) \
2887 { \
2888 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2889 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2890 } \
2891 (selValue) = 0; \
2892 }
2893#endif
2894
2895
2896/**
2897 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2898 * the host-state area in the VMCS.
2899 *
2900 * @returns VBox status code.
2901 * @param pVM Pointer to the VM.
2902 * @param pVCpu Pointer to the VMCPU.
2903 */
2904DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2905{
2906 int rc = VERR_INTERNAL_ERROR_5;
2907
2908#if HC_ARCH_BITS == 64
2909 /*
2910 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2911 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2912 */
2913 AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2914 ("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2915#endif
2916
2917 /*
2918 * Host DS, ES, FS and GS segment registers.
2919 */
2920#if HC_ARCH_BITS == 64
2921 RTSEL uSelDS = ASMGetDS();
2922 RTSEL uSelES = ASMGetES();
2923 RTSEL uSelFS = ASMGetFS();
2924 RTSEL uSelGS = ASMGetGS();
2925#else
2926 RTSEL uSelDS = 0;
2927 RTSEL uSelES = 0;
2928 RTSEL uSelFS = 0;
2929 RTSEL uSelGS = 0;
2930#endif
2931
2932 /* Recalculate which host-state bits need to be manually restored. */
2933 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2934
2935 /*
2936 * Host CS and SS segment registers.
2937 */
2938#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2939 RTSEL uSelCS;
2940 RTSEL uSelSS;
2941 if (HMVMX_IS_64BIT_HOST_MODE())
2942 {
2943 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2944 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2945 }
2946 else
2947 {
2948 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2949 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2950 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2951 }
2952#else
2953 RTSEL uSelCS = ASMGetCS();
2954 RTSEL uSelSS = ASMGetSS();
2955#endif
2956
2957 /*
2958 * Host TR segment register.
2959 */
2960 RTSEL uSelTR = ASMGetTR();
2961
2962#if HC_ARCH_BITS == 64
2963 /*
2964 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2965 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2966 */
2967 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2968 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2969 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2970 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2971# undef VMXLOCAL_ADJUST_HOST_SEG
2972#endif
2973
2974 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2975 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2976 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2977 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2978 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2979 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2980 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2981 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2982 Assert(uSelCS);
2983 Assert(uSelTR);
2984
2985 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2986#if 0
2987 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2988 Assert(uSelSS != 0);
2989#endif
2990
2991 /* Write these host selector fields into the host-state area in the VMCS. */
2992 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2993 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2994#if HC_ARCH_BITS == 64
2995 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2996 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2997 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2998 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2999#endif
3000 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
3001
3002 /*
3003 * Host GDTR and IDTR.
3004 */
3005 RTGDTR Gdtr;
3006 RT_ZERO(Gdtr);
3007#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3008 if (HMVMX_IS_64BIT_HOST_MODE())
3009 {
3010 X86XDTR64 Gdtr64;
3011 X86XDTR64 Idtr64;
3012 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
3013 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
3014 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
3015
3016 Gdtr.cbGdt = Gdtr64.cb;
3017 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
3018 }
3019 else
3020#endif
3021 {
3022 RTIDTR Idtr;
3023 ASMGetGDTR(&Gdtr);
3024 ASMGetIDTR(&Idtr);
3025 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
3026 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
3027
3028#if HC_ARCH_BITS == 64
3029 /*
3030 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3031 * maximum limit (0xffff) on every VM-exit.
3032 */
3033 if (Gdtr.cbGdt != 0xffff)
3034 {
3035 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3036 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3037 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3038 }
3039
3040 /*
3041 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3042 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3043 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3044 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3045 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3046 * hosts where we are pretty sure it won't cause trouble.
3047 */
3048# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3049 if (Idtr.cbIdt < 0x0fff)
3050# else
3051 if (Idtr.cbIdt != 0xffff)
3052# endif
3053 {
3054 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3055 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3056 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3057 }
3058#endif
3059 }
3060
3061 /*
3062 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3063 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3064 */
3065 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3066 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3067 VERR_VMX_INVALID_HOST_STATE);
3068
3069 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3070#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3071 if (HMVMX_IS_64BIT_HOST_MODE())
3072 {
3073 /* We need the 64-bit TR base for hybrid darwin. */
3074 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
3075 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
3076 }
3077 else
3078#endif
3079 {
3080 uintptr_t uTRBase;
3081#if HC_ARCH_BITS == 64
3082 uTRBase = X86DESC64_BASE(pDesc);
3083
3084 /*
3085 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3086 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3087 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3088 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3089 *
3090 * [1] See Intel spec. 3.5 "System Descriptor Types".
3091 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3092 */
3093 Assert(pDesc->System.u4Type == 11);
3094 if ( pDesc->System.u16LimitLow != 0x67
3095 || pDesc->System.u4LimitHigh)
3096 {
3097 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3098 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3099 if (pVM->hm.s.uHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3100 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3101 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3102
3103 /* Store the GDTR here as we need it while restoring TR. */
3104 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3105 }
3106#else
3107 uTRBase = X86DESC_BASE(pDesc);
3108#endif
3109 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3110 }
3111 AssertRCReturn(rc, rc);
3112
3113 /*
3114 * Host FS base and GS base.
3115 */
3116#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3117 if (HMVMX_IS_64BIT_HOST_MODE())
3118 {
3119 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3120 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3121 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
3122 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
3123
3124# if HC_ARCH_BITS == 64
3125 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3126 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3127 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3128 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3129 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3130# endif
3131 }
3132#endif
3133 return rc;
3134}
3135
3136
3137/**
3138 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3139 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3140 * the host after every successful VM-exit.
3141 *
3142 * @returns VBox status code.
3143 * @param pVM Pointer to the VM.
3144 * @param pVCpu Pointer to the VMCPU.
3145 *
3146 * @remarks No-long-jump zone!!!
3147 */
3148DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3149{
3150 NOREF(pVM);
3151
3152 AssertPtr(pVCpu);
3153 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3154
3155 int rc = VINF_SUCCESS;
3156#if HC_ARCH_BITS == 64
3157 if (pVM->hm.s.fAllow64BitGuests)
3158 hmR0VmxLazySaveHostMsrs(pVCpu);
3159#endif
3160
3161 /*
3162 * Host Sysenter MSRs.
3163 */
3164 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3165 AssertRCReturn(rc, rc);
3166#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3167 if (HMVMX_IS_64BIT_HOST_MODE())
3168 {
3169 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3170 AssertRCReturn(rc, rc);
3171 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3172 }
3173 else
3174 {
3175 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3176 AssertRCReturn(rc, rc);
3177 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3178 }
3179#elif HC_ARCH_BITS == 32
3180 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3181 AssertRCReturn(rc, rc);
3182 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3183#else
3184 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3185 AssertRCReturn(rc, rc);
3186 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3187#endif
3188 AssertRCReturn(rc, rc);
3189
3190 /*
3191 * Host EFER MSR.
3192 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3193 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3194 */
3195 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3196 {
3197 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3198 AssertRCReturn(rc, rc);
3199 }
3200
3201 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3202 * hmR0VmxLoadGuestExitCtls() !! */
3203
3204 return rc;
3205}
3206
3207
3208/**
3209 * Figures out if we need to swap the EFER MSR which is
3210 * particularly expensive.
3211 *
3212 * We check all relevant bits. For now, that's everything
3213 * besides LMA/LME, as these two bits are handled by VM-entry,
3214 * see hmR0VmxLoadGuestExitCtls() and
3215 * hmR0VMxLoadGuestEntryCtls().
3216 *
3217 * @returns true if we need to load guest EFER, false otherwise.
3218 * @param pVCpu Pointer to the VMCPU.
3219 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3220 * out-of-sync. Make sure to update the required fields
3221 * before using them.
3222 *
3223 * @remarks Requires EFER, CR4.
3224 * @remarks No-long-jump zone!!!
3225 */
3226static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3227{
3228#ifdef HMVMX_ALWAYS_SWAP_EFER
3229 return true;
3230#endif
3231
3232#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3233 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3234 if (CPUMIsGuestInLongMode(pVCpu))
3235 return false;
3236#endif
3237
3238 PVM pVM = pVCpu->CTX_SUFF(pVM);
3239 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3240 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3241
3242 /*
3243 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3244 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3245 */
3246 if ( CPUMIsGuestInLongMode(pVCpu)
3247 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3248 {
3249 return true;
3250 }
3251
3252 /*
3253 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3254 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3255 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3256 */
3257 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3258 && (pMixedCtx->cr0 & X86_CR0_PG)
3259 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3260 {
3261 /* Assert that host is PAE capable. */
3262 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3263 return true;
3264 }
3265
3266 /** @todo Check the latest Intel spec. for any other bits,
3267 * like SMEP/SMAP? */
3268 return false;
3269}
3270
3271
3272/**
3273 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3274 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3275 * controls".
3276 *
3277 * @returns VBox status code.
3278 * @param pVCpu Pointer to the VMCPU.
3279 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3280 * out-of-sync. Make sure to update the required fields
3281 * before using them.
3282 *
3283 * @remarks Requires EFER.
3284 * @remarks No-long-jump zone!!!
3285 */
3286DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3287{
3288 int rc = VINF_SUCCESS;
3289 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3290 {
3291 PVM pVM = pVCpu->CTX_SUFF(pVM);
3292 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3293 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3294
3295 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3296 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3297
3298 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3299 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3300 {
3301 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3302 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3303 }
3304 else
3305 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3306
3307 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3308 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3309 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3310 {
3311 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3312 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3313 }
3314
3315 /*
3316 * The following should -not- be set (since we're not in SMM mode):
3317 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3318 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3319 */
3320
3321 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3322 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3323
3324 if ((val & zap) != val)
3325 {
3326 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3327 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3328 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3329 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3330 }
3331
3332 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3333 AssertRCReturn(rc, rc);
3334
3335 pVCpu->hm.s.vmx.u32EntryCtls = val;
3336 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3337 }
3338 return rc;
3339}
3340
3341
3342/**
3343 * Sets up the VM-exit controls in the VMCS.
3344 *
3345 * @returns VBox status code.
3346 * @param pVM Pointer to the VM.
3347 * @param pVCpu Pointer to the VMCPU.
3348 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3349 * out-of-sync. Make sure to update the required fields
3350 * before using them.
3351 *
3352 * @remarks Requires EFER.
3353 */
3354DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3355{
3356 NOREF(pMixedCtx);
3357
3358 int rc = VINF_SUCCESS;
3359 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3360 {
3361 PVM pVM = pVCpu->CTX_SUFF(pVM);
3362 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3363 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3364
3365 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3366 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3367
3368 /*
3369 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3370 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3371 */
3372#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3373 if (HMVMX_IS_64BIT_HOST_MODE())
3374 {
3375 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3376 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3377 }
3378 else
3379 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3380#else
3381 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3382 {
3383 /* The switcher returns to long mode, EFER is managed by the switcher. */
3384 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3385 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3386 }
3387 else
3388 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3389#endif /* HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
3390
3391 /* If the newer VMCS fields for managing EFER exists, use it. */
3392 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3393 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3394 {
3395 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3396 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3397 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3398 }
3399
3400 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3401 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3402
3403 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3404 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3405 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3406
3407 if ( pVM->hm.s.vmx.fUsePreemptTimer
3408 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3409 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3410
3411 if ((val & zap) != val)
3412 {
3413 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3414 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3415 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3416 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3417 }
3418
3419 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3420 AssertRCReturn(rc, rc);
3421
3422 pVCpu->hm.s.vmx.u32ExitCtls = val;
3423 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3424 }
3425 return rc;
3426}
3427
3428
3429/**
3430 * Loads the guest APIC and related state.
3431 *
3432 * @returns VBox status code.
3433 * @param pVM Pointer to the VM.
3434 * @param pVCpu Pointer to the VMCPU.
3435 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3436 * out-of-sync. Make sure to update the required fields
3437 * before using them.
3438 */
3439DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3440{
3441 NOREF(pMixedCtx);
3442
3443 int rc = VINF_SUCCESS;
3444 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3445 {
3446 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3447 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3448 {
3449 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3450
3451 bool fPendingIntr = false;
3452 uint8_t u8Tpr = 0;
3453 uint8_t u8PendingIntr = 0;
3454 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3455 AssertRCReturn(rc, rc);
3456
3457 /*
3458 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3459 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3460 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3461 * the interrupt when we VM-exit for other reasons.
3462 */
3463 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3464 uint32_t u32TprThreshold = 0;
3465 if (fPendingIntr)
3466 {
3467 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3468 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3469 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3470 if (u8PendingPriority <= u8TprPriority)
3471 u32TprThreshold = u8PendingPriority;
3472 else
3473 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3474 }
3475 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3476
3477 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3478 AssertRCReturn(rc, rc);
3479 }
3480
3481 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3482 }
3483 return rc;
3484}
3485
3486
3487/**
3488 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3489 *
3490 * @returns Guest's interruptibility-state.
3491 * @param pVCpu Pointer to the VMCPU.
3492 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3493 * out-of-sync. Make sure to update the required fields
3494 * before using them.
3495 *
3496 * @remarks No-long-jump zone!!!
3497 */
3498DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3499{
3500 /*
3501 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3502 */
3503 uint32_t uIntrState = 0;
3504 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3505 {
3506 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3507 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3508 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3509 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3510 {
3511 if (pMixedCtx->eflags.Bits.u1IF)
3512 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3513 else
3514 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3515 }
3516 /* else: Although we can clear the force-flag here, let's keep this side-effects free. */
3517 }
3518
3519 /*
3520 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3521 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3522 * setting this would block host-NMIs and IRET will not clear the blocking.
3523 *
3524 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3525 */
3526 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3527 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3528 {
3529 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3530 }
3531
3532 return uIntrState;
3533}
3534
3535
3536/**
3537 * Loads the guest's interruptibility-state into the guest-state area in the
3538 * VMCS.
3539 *
3540 * @returns VBox status code.
3541 * @param pVCpu Pointer to the VMCPU.
3542 * @param uIntrState The interruptibility-state to set.
3543 */
3544static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3545{
3546 NOREF(pVCpu);
3547 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3548 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3549 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3550 AssertRCReturn(rc, rc);
3551 return rc;
3552}
3553
3554
3555/**
3556 * Loads the guest's RIP into the guest-state area in the VMCS.
3557 *
3558 * @returns VBox status code.
3559 * @param pVCpu Pointer to the VMCPU.
3560 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3561 * out-of-sync. Make sure to update the required fields
3562 * before using them.
3563 *
3564 * @remarks No-long-jump zone!!!
3565 */
3566static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3567{
3568 int rc = VINF_SUCCESS;
3569 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3570 {
3571 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3572 AssertRCReturn(rc, rc);
3573
3574 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3575 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3576 HMCPU_CF_VALUE(pVCpu)));
3577 }
3578 return rc;
3579}
3580
3581
3582/**
3583 * Loads the guest's RSP into the guest-state area in the VMCS.
3584 *
3585 * @returns VBox status code.
3586 * @param pVCpu Pointer to the VMCPU.
3587 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3588 * out-of-sync. Make sure to update the required fields
3589 * before using them.
3590 *
3591 * @remarks No-long-jump zone!!!
3592 */
3593static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3594{
3595 int rc = VINF_SUCCESS;
3596 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3597 {
3598 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3599 AssertRCReturn(rc, rc);
3600
3601 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3602 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3603 }
3604 return rc;
3605}
3606
3607
3608/**
3609 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3610 *
3611 * @returns VBox status code.
3612 * @param pVCpu Pointer to the VMCPU.
3613 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3614 * out-of-sync. Make sure to update the required fields
3615 * before using them.
3616 *
3617 * @remarks No-long-jump zone!!!
3618 */
3619static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3620{
3621 int rc = VINF_SUCCESS;
3622 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3623 {
3624 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3625 Let us assert it as such and use 32-bit VMWRITE. */
3626 Assert(!(pMixedCtx->rflags.u64 >> 32));
3627 X86EFLAGS Eflags = pMixedCtx->eflags;
3628 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3629 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3630 * These will never be cleared/set, unless some other part of the VMM
3631 * code is buggy - in which case we're better of finding and fixing
3632 * those bugs than hiding them. */
3633 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3634 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3635 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3636 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3637
3638 /*
3639 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3640 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3641 */
3642 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3643 {
3644 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3645 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3646 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3647 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3648 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3649 }
3650
3651 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3652 AssertRCReturn(rc, rc);
3653
3654 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3655 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3656 }
3657 return rc;
3658}
3659
3660
3661/**
3662 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3663 *
3664 * @returns VBox status code.
3665 * @param pVCpu Pointer to the VMCPU.
3666 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3667 * out-of-sync. Make sure to update the required fields
3668 * before using them.
3669 *
3670 * @remarks No-long-jump zone!!!
3671 */
3672DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3673{
3674 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3675 AssertRCReturn(rc, rc);
3676 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3677 AssertRCReturn(rc, rc);
3678 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3679 AssertRCReturn(rc, rc);
3680 return rc;
3681}
3682
3683
3684/**
3685 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3686 * CR0 is partially shared with the host and we have to consider the FPU bits.
3687 *
3688 * @returns VBox status code.
3689 * @param pVM Pointer to the VM.
3690 * @param pVCpu Pointer to the VMCPU.
3691 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3692 * out-of-sync. Make sure to update the required fields
3693 * before using them.
3694 *
3695 * @remarks No-long-jump zone!!!
3696 */
3697static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3698{
3699 /*
3700 * Guest CR0.
3701 * Guest FPU.
3702 */
3703 int rc = VINF_SUCCESS;
3704 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3705 {
3706 Assert(!(pMixedCtx->cr0 >> 32));
3707 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3708 PVM pVM = pVCpu->CTX_SUFF(pVM);
3709
3710 /* The guest's view (read access) of its CR0 is unblemished. */
3711 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3712 AssertRCReturn(rc, rc);
3713 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3714
3715 /* Setup VT-x's view of the guest CR0. */
3716 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3717 if (pVM->hm.s.fNestedPaging)
3718 {
3719 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3720 {
3721 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3722 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3723 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3724 }
3725 else
3726 {
3727 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3728 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3729 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3730 }
3731
3732 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3733 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3734 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3735
3736 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3737 AssertRCReturn(rc, rc);
3738 }
3739 else
3740 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3741
3742 /*
3743 * Guest FPU bits.
3744 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3745 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3746 */
3747 u32GuestCR0 |= X86_CR0_NE;
3748 bool fInterceptNM = false;
3749 if (CPUMIsGuestFPUStateActive(pVCpu))
3750 {
3751 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3752 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3753 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3754 }
3755 else
3756 {
3757 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3758 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3759 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3760 }
3761
3762 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3763 bool fInterceptMF = false;
3764 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3765 fInterceptMF = true;
3766
3767 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3768 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3769 {
3770 Assert(PDMVmmDevHeapIsEnabled(pVM));
3771 Assert(pVM->hm.s.vmx.pRealModeTSS);
3772 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3773 fInterceptNM = true;
3774 fInterceptMF = true;
3775 }
3776 else
3777 {
3778 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626} comment #11. */
3779 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3780 }
3781
3782 if (fInterceptNM)
3783 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3784 else
3785 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3786
3787 if (fInterceptMF)
3788 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3789 else
3790 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3791
3792 /* Additional intercepts for debugging, define these yourself explicitly. */
3793#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3794 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3795 | RT_BIT(X86_XCPT_BP)
3796 | RT_BIT(X86_XCPT_DB)
3797 | RT_BIT(X86_XCPT_DE)
3798 | RT_BIT(X86_XCPT_NM)
3799 | RT_BIT(X86_XCPT_TS)
3800 | RT_BIT(X86_XCPT_UD)
3801 | RT_BIT(X86_XCPT_NP)
3802 | RT_BIT(X86_XCPT_SS)
3803 | RT_BIT(X86_XCPT_GP)
3804 | RT_BIT(X86_XCPT_PF)
3805 | RT_BIT(X86_XCPT_MF)
3806 ;
3807#elif defined(HMVMX_ALWAYS_TRAP_PF)
3808 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3809#endif
3810
3811 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3812
3813 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3814 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3815 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3816 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3817 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3818 else
3819 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3820
3821 u32GuestCR0 |= uSetCR0;
3822 u32GuestCR0 &= uZapCR0;
3823 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3824
3825 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3826 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3827 AssertRCReturn(rc, rc);
3828 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3829 AssertRCReturn(rc, rc);
3830 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3831 uZapCR0));
3832
3833 /*
3834 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3835 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3836 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3837 */
3838 uint32_t u32CR0Mask = 0;
3839 u32CR0Mask = X86_CR0_PE
3840 | X86_CR0_NE
3841 | X86_CR0_WP
3842 | X86_CR0_PG
3843 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3844 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3845 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3846
3847 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3848 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3849 * and @bugref{6944}. */
3850#if 0
3851 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3852 u32CR0Mask &= ~X86_CR0_PE;
3853#endif
3854 if (pVM->hm.s.fNestedPaging)
3855 u32CR0Mask &= ~X86_CR0_WP;
3856
3857 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3858 if (fInterceptNM)
3859 {
3860 u32CR0Mask |= X86_CR0_TS
3861 | X86_CR0_MP;
3862 }
3863
3864 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3865 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3866 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3867 AssertRCReturn(rc, rc);
3868 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3869
3870 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3871 }
3872 return rc;
3873}
3874
3875
3876/**
3877 * Loads the guest control registers (CR3, CR4) into the guest-state area
3878 * in the VMCS.
3879 *
3880 * @returns VBox status code.
3881 * @param pVM Pointer to the VM.
3882 * @param pVCpu Pointer to the VMCPU.
3883 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3884 * out-of-sync. Make sure to update the required fields
3885 * before using them.
3886 *
3887 * @remarks No-long-jump zone!!!
3888 */
3889static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3890{
3891 int rc = VINF_SUCCESS;
3892 PVM pVM = pVCpu->CTX_SUFF(pVM);
3893
3894 /*
3895 * Guest CR2.
3896 * It's always loaded in the assembler code. Nothing to do here.
3897 */
3898
3899 /*
3900 * Guest CR3.
3901 */
3902 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3903 {
3904 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3905 if (pVM->hm.s.fNestedPaging)
3906 {
3907 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3908
3909 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3910 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3911 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3912 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3913
3914 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3915 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3916 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3917
3918 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3919 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3920 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3921 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3922
3923 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3924 AssertRCReturn(rc, rc);
3925 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3926
3927 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3928 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3929 {
3930 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3931 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3932 {
3933 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3934 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3935 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3936 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3937 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3938 }
3939
3940 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3941 have Unrestricted Execution to handle the guest when it's not using paging. */
3942 GCPhysGuestCR3 = pMixedCtx->cr3;
3943 }
3944 else
3945 {
3946 /*
3947 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3948 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3949 * EPT takes care of translating it to host-physical addresses.
3950 */
3951 RTGCPHYS GCPhys;
3952 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3953 Assert(PDMVmmDevHeapIsEnabled(pVM));
3954
3955 /* We obtain it here every time as the guest could have relocated this PCI region. */
3956 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3957 AssertRCReturn(rc, rc);
3958
3959 GCPhysGuestCR3 = GCPhys;
3960 }
3961
3962 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
3963 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3964 }
3965 else
3966 {
3967 /* Non-nested paging case, just use the hypervisor's CR3. */
3968 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3969
3970 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
3971 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3972 }
3973 AssertRCReturn(rc, rc);
3974
3975 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3976 }
3977
3978 /*
3979 * Guest CR4.
3980 */
3981 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3982 {
3983 Assert(!(pMixedCtx->cr4 >> 32));
3984 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3985
3986 /* The guest's view of its CR4 is unblemished. */
3987 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3988 AssertRCReturn(rc, rc);
3989 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
3990
3991 /* Setup VT-x's view of the guest CR4. */
3992 /*
3993 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3994 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3995 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3996 */
3997 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3998 {
3999 Assert(pVM->hm.s.vmx.pRealModeTSS);
4000 Assert(PDMVmmDevHeapIsEnabled(pVM));
4001 u32GuestCR4 &= ~X86_CR4_VME;
4002 }
4003
4004 if (pVM->hm.s.fNestedPaging)
4005 {
4006 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4007 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4008 {
4009 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4010 u32GuestCR4 |= X86_CR4_PSE;
4011 /* Our identity mapping is a 32-bit page directory. */
4012 u32GuestCR4 &= ~X86_CR4_PAE;
4013 }
4014 /* else use guest CR4.*/
4015 }
4016 else
4017 {
4018 /*
4019 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4020 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4021 */
4022 switch (pVCpu->hm.s.enmShadowMode)
4023 {
4024 case PGMMODE_REAL: /* Real-mode. */
4025 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4026 case PGMMODE_32_BIT: /* 32-bit paging. */
4027 {
4028 u32GuestCR4 &= ~X86_CR4_PAE;
4029 break;
4030 }
4031
4032 case PGMMODE_PAE: /* PAE paging. */
4033 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4034 {
4035 u32GuestCR4 |= X86_CR4_PAE;
4036 break;
4037 }
4038
4039 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4040 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4041#ifdef VBOX_ENABLE_64_BITS_GUESTS
4042 break;
4043#endif
4044 default:
4045 AssertFailed();
4046 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4047 }
4048 }
4049
4050 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4051 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4052 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4053 u32GuestCR4 |= uSetCR4;
4054 u32GuestCR4 &= uZapCR4;
4055
4056 /* Write VT-x's view of the guest CR4 into the VMCS. */
4057 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4058 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4059 AssertRCReturn(rc, rc);
4060
4061 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4062 uint32_t u32CR4Mask = 0;
4063 u32CR4Mask = X86_CR4_VME
4064 | X86_CR4_PAE
4065 | X86_CR4_PGE
4066 | X86_CR4_PSE
4067 | X86_CR4_VMXE;
4068 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4069 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4070 AssertRCReturn(rc, rc);
4071
4072 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4073 }
4074 return rc;
4075}
4076
4077
4078/**
4079 * Loads the guest debug registers into the guest-state area in the VMCS.
4080 * This also sets up whether #DB and MOV DRx accesses cause VM-exits.
4081 *
4082 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4083 *
4084 * @returns VBox status code.
4085 * @param pVCpu Pointer to the VMCPU.
4086 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4087 * out-of-sync. Make sure to update the required fields
4088 * before using them.
4089 *
4090 * @remarks No-long-jump zone!!!
4091 */
4092static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4093{
4094 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4095 return VINF_SUCCESS;
4096
4097#ifdef VBOX_STRICT
4098 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4099 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4100 {
4101 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4102 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4103 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4104 }
4105#endif
4106
4107 int rc;
4108 PVM pVM = pVCpu->CTX_SUFF(pVM);
4109 bool fInterceptDB = false;
4110 bool fInterceptMovDRx = false;
4111 if ( pVCpu->hm.s.fSingleInstruction
4112 || DBGFIsStepping(pVCpu))
4113 {
4114 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4115 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4116 {
4117 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4118 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4119 AssertRCReturn(rc, rc);
4120 Assert(fInterceptDB == false);
4121 }
4122 else
4123 {
4124 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4125 pVCpu->hm.s.fClearTrapFlag = true;
4126 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4127 fInterceptDB = true;
4128 }
4129 }
4130
4131 if ( fInterceptDB
4132 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4133 {
4134 /*
4135 * Use the combined guest and host DRx values found in the hypervisor
4136 * register set because the debugger has breakpoints active or someone
4137 * is single stepping on the host side without a monitor trap flag.
4138 *
4139 * Note! DBGF expects a clean DR6 state before executing guest code.
4140 */
4141#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4142 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4143 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4144 {
4145 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4146 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4147 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4148 }
4149 else
4150#endif
4151 if (!CPUMIsHyperDebugStateActive(pVCpu))
4152 {
4153 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4154 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4155 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4156 }
4157
4158 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4159 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4160 AssertRCReturn(rc, rc);
4161
4162 pVCpu->hm.s.fUsingHyperDR7 = true;
4163 fInterceptDB = true;
4164 fInterceptMovDRx = true;
4165 }
4166 else
4167 {
4168 /*
4169 * If the guest has enabled debug registers, we need to load them prior to
4170 * executing guest code so they'll trigger at the right time.
4171 */
4172 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4173 {
4174#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4175 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4176 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4177 {
4178 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4179 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4180 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4181 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4182 }
4183 else
4184#endif
4185 if (!CPUMIsGuestDebugStateActive(pVCpu))
4186 {
4187 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4188 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4189 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4190 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4191 }
4192 Assert(!fInterceptDB);
4193 Assert(!fInterceptMovDRx);
4194 }
4195 /*
4196 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4197 * must intercept #DB in order to maintain a correct DR6 guest value.
4198 */
4199#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4200 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4201 && !CPUMIsGuestDebugStateActive(pVCpu))
4202#else
4203 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4204#endif
4205 {
4206 fInterceptMovDRx = true;
4207 fInterceptDB = true;
4208 }
4209
4210 /* Update guest DR7. */
4211 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4212 AssertRCReturn(rc, rc);
4213
4214 pVCpu->hm.s.fUsingHyperDR7 = false;
4215 }
4216
4217 /*
4218 * Update the exception bitmap regarding intercepting #DB generated by the guest.
4219 */
4220 if ( fInterceptDB
4221 || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4222 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
4223 else
4224 {
4225#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4226 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
4227#endif
4228 }
4229 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
4230 AssertRCReturn(rc, rc);
4231
4232 /*
4233 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4234 */
4235 if (fInterceptMovDRx)
4236 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4237 else
4238 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4239 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4240 AssertRCReturn(rc, rc);
4241
4242 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4243 return VINF_SUCCESS;
4244}
4245
4246
4247#ifdef VBOX_STRICT
4248/**
4249 * Strict function to validate segment registers.
4250 *
4251 * @remarks ASSUMES CR0 is up to date.
4252 */
4253static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4254{
4255 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4256 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4257 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4258 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4259 && ( !CPUMIsGuestInRealModeEx(pCtx)
4260 && !CPUMIsGuestInV86ModeEx(pCtx)))
4261 {
4262 /* Protected mode checks */
4263 /* CS */
4264 Assert(pCtx->cs.Attr.n.u1Present);
4265 Assert(!(pCtx->cs.Attr.u & 0xf00));
4266 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4267 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4268 || !(pCtx->cs.Attr.n.u1Granularity));
4269 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4270 || (pCtx->cs.Attr.n.u1Granularity));
4271 /* CS cannot be loaded with NULL in protected mode. */
4272 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4273 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4274 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4275 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4276 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4277 else
4278 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4279 /* SS */
4280 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4281 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4282 if ( !(pCtx->cr0 & X86_CR0_PE)
4283 || pCtx->cs.Attr.n.u4Type == 3)
4284 {
4285 Assert(!pCtx->ss.Attr.n.u2Dpl);
4286 }
4287 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4288 {
4289 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4290 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4291 Assert(pCtx->ss.Attr.n.u1Present);
4292 Assert(!(pCtx->ss.Attr.u & 0xf00));
4293 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4294 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4295 || !(pCtx->ss.Attr.n.u1Granularity));
4296 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4297 || (pCtx->ss.Attr.n.u1Granularity));
4298 }
4299 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4300 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4301 {
4302 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4303 Assert(pCtx->ds.Attr.n.u1Present);
4304 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4305 Assert(!(pCtx->ds.Attr.u & 0xf00));
4306 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4307 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4308 || !(pCtx->ds.Attr.n.u1Granularity));
4309 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4310 || (pCtx->ds.Attr.n.u1Granularity));
4311 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4312 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4313 }
4314 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4315 {
4316 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4317 Assert(pCtx->es.Attr.n.u1Present);
4318 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4319 Assert(!(pCtx->es.Attr.u & 0xf00));
4320 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4321 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4322 || !(pCtx->es.Attr.n.u1Granularity));
4323 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4324 || (pCtx->es.Attr.n.u1Granularity));
4325 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4326 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4327 }
4328 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4329 {
4330 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4331 Assert(pCtx->fs.Attr.n.u1Present);
4332 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4333 Assert(!(pCtx->fs.Attr.u & 0xf00));
4334 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4335 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4336 || !(pCtx->fs.Attr.n.u1Granularity));
4337 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4338 || (pCtx->fs.Attr.n.u1Granularity));
4339 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4340 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4341 }
4342 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4343 {
4344 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4345 Assert(pCtx->gs.Attr.n.u1Present);
4346 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4347 Assert(!(pCtx->gs.Attr.u & 0xf00));
4348 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4349 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4350 || !(pCtx->gs.Attr.n.u1Granularity));
4351 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4352 || (pCtx->gs.Attr.n.u1Granularity));
4353 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4354 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4355 }
4356 /* 64-bit capable CPUs. */
4357# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4358 if (HMVMX_IS_64BIT_HOST_MODE())
4359 {
4360 Assert(!(pCtx->cs.u64Base >> 32));
4361 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4362 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4363 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4364 }
4365# endif
4366 }
4367 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4368 || ( CPUMIsGuestInRealModeEx(pCtx)
4369 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4370 {
4371 /* Real and v86 mode checks. */
4372 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4373 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4374 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4375 {
4376 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4377 }
4378 else
4379 {
4380 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4381 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4382 }
4383
4384 /* CS */
4385 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4386 Assert(pCtx->cs.u32Limit == 0xffff);
4387 Assert(u32CSAttr == 0xf3);
4388 /* SS */
4389 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4390 Assert(pCtx->ss.u32Limit == 0xffff);
4391 Assert(u32SSAttr == 0xf3);
4392 /* DS */
4393 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4394 Assert(pCtx->ds.u32Limit == 0xffff);
4395 Assert(u32DSAttr == 0xf3);
4396 /* ES */
4397 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4398 Assert(pCtx->es.u32Limit == 0xffff);
4399 Assert(u32ESAttr == 0xf3);
4400 /* FS */
4401 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4402 Assert(pCtx->fs.u32Limit == 0xffff);
4403 Assert(u32FSAttr == 0xf3);
4404 /* GS */
4405 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4406 Assert(pCtx->gs.u32Limit == 0xffff);
4407 Assert(u32GSAttr == 0xf3);
4408 /* 64-bit capable CPUs. */
4409# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4410 if (HMVMX_IS_64BIT_HOST_MODE())
4411 {
4412 Assert(!(pCtx->cs.u64Base >> 32));
4413 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4414 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4415 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4416 }
4417# endif
4418 }
4419}
4420#endif /* VBOX_STRICT */
4421
4422
4423/**
4424 * Writes a guest segment register into the guest-state area in the VMCS.
4425 *
4426 * @returns VBox status code.
4427 * @param pVCpu Pointer to the VMCPU.
4428 * @param idxSel Index of the selector in the VMCS.
4429 * @param idxLimit Index of the segment limit in the VMCS.
4430 * @param idxBase Index of the segment base in the VMCS.
4431 * @param idxAccess Index of the access rights of the segment in the VMCS.
4432 * @param pSelReg Pointer to the segment selector.
4433 *
4434 * @remarks No-long-jump zone!!!
4435 */
4436static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4437 uint32_t idxAccess, PCPUMSELREG pSelReg)
4438{
4439 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4440 AssertRCReturn(rc, rc);
4441 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4442 AssertRCReturn(rc, rc);
4443 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4444 AssertRCReturn(rc, rc);
4445
4446 uint32_t u32Access = pSelReg->Attr.u;
4447 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4448 {
4449 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4450 u32Access = 0xf3;
4451 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4452 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4453 }
4454 else
4455 {
4456 /*
4457 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4458 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4459 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4460 * loaded in protected-mode have their attribute as 0.
4461 */
4462 if (!u32Access)
4463 u32Access = X86DESCATTR_UNUSABLE;
4464 }
4465
4466 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4467 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4468 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4469
4470 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4471 AssertRCReturn(rc, rc);
4472 return rc;
4473}
4474
4475
4476/**
4477 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4478 * into the guest-state area in the VMCS.
4479 *
4480 * @returns VBox status code.
4481 * @param pVM Pointer to the VM.
4482 * @param pVCPU Pointer to the VMCPU.
4483 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4484 * out-of-sync. Make sure to update the required fields
4485 * before using them.
4486 *
4487 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4488 * @remarks No-long-jump zone!!!
4489 */
4490static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4491{
4492 int rc = VERR_INTERNAL_ERROR_5;
4493 PVM pVM = pVCpu->CTX_SUFF(pVM);
4494
4495 /*
4496 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4497 */
4498 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4499 {
4500 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4501 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4502 {
4503 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4504 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4505 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4506 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4507 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4508 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4509 }
4510
4511#ifdef VBOX_WITH_REM
4512 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4513 {
4514 Assert(pVM->hm.s.vmx.pRealModeTSS);
4515 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4516 if ( pVCpu->hm.s.vmx.fWasInRealMode
4517 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4518 {
4519 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4520 in real-mode (e.g. OpenBSD 4.0) */
4521 REMFlushTBs(pVM);
4522 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4523 pVCpu->hm.s.vmx.fWasInRealMode = false;
4524 }
4525 }
4526#endif
4527 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4528 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4529 AssertRCReturn(rc, rc);
4530 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4531 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4532 AssertRCReturn(rc, rc);
4533 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4534 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4535 AssertRCReturn(rc, rc);
4536 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4537 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4538 AssertRCReturn(rc, rc);
4539 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4540 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4541 AssertRCReturn(rc, rc);
4542 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4543 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4544 AssertRCReturn(rc, rc);
4545
4546#ifdef VBOX_STRICT
4547 /* Validate. */
4548 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4549#endif
4550
4551 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4552 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4553 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4554 }
4555
4556 /*
4557 * Guest TR.
4558 */
4559 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4560 {
4561 /*
4562 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4563 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4564 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4565 */
4566 uint16_t u16Sel = 0;
4567 uint32_t u32Limit = 0;
4568 uint64_t u64Base = 0;
4569 uint32_t u32AccessRights = 0;
4570
4571 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4572 {
4573 u16Sel = pMixedCtx->tr.Sel;
4574 u32Limit = pMixedCtx->tr.u32Limit;
4575 u64Base = pMixedCtx->tr.u64Base;
4576 u32AccessRights = pMixedCtx->tr.Attr.u;
4577 }
4578 else
4579 {
4580 Assert(pVM->hm.s.vmx.pRealModeTSS);
4581 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4582
4583 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4584 RTGCPHYS GCPhys;
4585 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4586 AssertRCReturn(rc, rc);
4587
4588 X86DESCATTR DescAttr;
4589 DescAttr.u = 0;
4590 DescAttr.n.u1Present = 1;
4591 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4592
4593 u16Sel = 0;
4594 u32Limit = HM_VTX_TSS_SIZE;
4595 u64Base = GCPhys; /* in real-mode phys = virt. */
4596 u32AccessRights = DescAttr.u;
4597 }
4598
4599 /* Validate. */
4600 Assert(!(u16Sel & RT_BIT(2)));
4601 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4602 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4603 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4604 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4605 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4606 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4607 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4608 Assert( (u32Limit & 0xfff) == 0xfff
4609 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4610 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4611 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4612
4613 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4614 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4615 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4616 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4617
4618 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4619 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4620 }
4621
4622 /*
4623 * Guest GDTR.
4624 */
4625 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4626 {
4627 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4628 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4629
4630 /* Validate. */
4631 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4632
4633 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4634 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4635 }
4636
4637 /*
4638 * Guest LDTR.
4639 */
4640 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4641 {
4642 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4643 uint32_t u32Access = 0;
4644 if (!pMixedCtx->ldtr.Attr.u)
4645 u32Access = X86DESCATTR_UNUSABLE;
4646 else
4647 u32Access = pMixedCtx->ldtr.Attr.u;
4648
4649 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4650 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4651 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4652 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4653
4654 /* Validate. */
4655 if (!(u32Access & X86DESCATTR_UNUSABLE))
4656 {
4657 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4658 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4659 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4660 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4661 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4662 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4663 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4664 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4665 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4666 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4667 }
4668
4669 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4670 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4671 }
4672
4673 /*
4674 * Guest IDTR.
4675 */
4676 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4677 {
4678 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4679 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4680
4681 /* Validate. */
4682 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4683
4684 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4685 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4686 }
4687
4688 return VINF_SUCCESS;
4689}
4690
4691
4692/**
4693 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4694 * areas.
4695 *
4696 * These MSRs will automatically be loaded to the host CPU on every successful
4697 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4698 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4699 * -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4700 *
4701 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4702 *
4703 * @returns VBox status code.
4704 * @param pVCpu Pointer to the VMCPU.
4705 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4706 * out-of-sync. Make sure to update the required fields
4707 * before using them.
4708 *
4709 * @remarks No-long-jump zone!!!
4710 */
4711static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4712{
4713 AssertPtr(pVCpu);
4714 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4715
4716 /*
4717 * MSRs that we use the auto-load/store MSR area in the VMCS.
4718 */
4719 PVM pVM = pVCpu->CTX_SUFF(pVM);
4720 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4721 {
4722 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4723#if HC_ARCH_BITS == 32 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4724 if (pVM->hm.s.fAllow64BitGuests)
4725 {
4726 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4727 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4728 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4729 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4730# ifdef DEBUG
4731 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4732 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4733 {
4734 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4735 pMsr->u64Value));
4736 }
4737# endif
4738 }
4739#endif
4740 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4741 }
4742
4743 /*
4744 * Guest Sysenter MSRs.
4745 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4746 * VM-exits on WRMSRs for these MSRs.
4747 */
4748 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4749 {
4750 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4751 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4752 }
4753
4754 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4755 {
4756 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4757 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4758 }
4759
4760 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4761 {
4762 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4763 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4764 }
4765
4766 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4767 {
4768 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4769 {
4770 /*
4771 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4772 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4773 */
4774 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4775 {
4776 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4777 AssertRCReturn(rc,rc);
4778 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4779 }
4780 else
4781 {
4782 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */);
4783 /* We need to intercept reads too, see @bugref{7386} comment #16. */
4784 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4785 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4786 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4787 }
4788 }
4789 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4790 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4791 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4792 }
4793
4794 return VINF_SUCCESS;
4795}
4796
4797
4798/**
4799 * Loads the guest activity state into the guest-state area in the VMCS.
4800 *
4801 * @returns VBox status code.
4802 * @param pVCpu Pointer to the VMCPU.
4803 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4804 * out-of-sync. Make sure to update the required fields
4805 * before using them.
4806 *
4807 * @remarks No-long-jump zone!!!
4808 */
4809static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4810{
4811 NOREF(pCtx);
4812 /** @todo See if we can make use of other states, e.g.
4813 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4814 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4815 {
4816 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4817 AssertRCReturn(rc, rc);
4818
4819 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4820 }
4821 return VINF_SUCCESS;
4822}
4823
4824
4825/**
4826 * Sets up the appropriate function to run guest code.
4827 *
4828 * @returns VBox status code.
4829 * @param pVCpu Pointer to the VMCPU.
4830 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4831 * out-of-sync. Make sure to update the required fields
4832 * before using them.
4833 *
4834 * @remarks No-long-jump zone!!!
4835 */
4836static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4837{
4838 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4839 {
4840#ifndef VBOX_ENABLE_64_BITS_GUESTS
4841 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4842#endif
4843 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4844#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4845 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4846 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4847 {
4848 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4849 {
4850 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4851 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4852 | HM_CHANGED_VMX_ENTRY_CTLS
4853 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4854 }
4855 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4856 }
4857#else
4858 /* 64-bit host or hybrid host. */
4859 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4860#endif
4861 }
4862 else
4863 {
4864 /* Guest is not in long mode, use the 32-bit handler. */
4865#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4866 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4867 {
4868 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4869 {
4870 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4871 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4872 | HM_CHANGED_VMX_ENTRY_CTLS
4873 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4874 }
4875 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4876 }
4877#else
4878 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4879#endif
4880 }
4881 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4882 return VINF_SUCCESS;
4883}
4884
4885
4886/**
4887 * Wrapper for running the guest code in VT-x.
4888 *
4889 * @returns VBox strict status code.
4890 * @param pVM Pointer to the VM.
4891 * @param pVCpu Pointer to the VMCPU.
4892 * @param pCtx Pointer to the guest-CPU context.
4893 *
4894 * @remarks No-long-jump zone!!!
4895 */
4896DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4897{
4898 /*
4899 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4900 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4901 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4902 */
4903 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4904 /** @todo Add stats for resume vs launch. */
4905#ifdef VBOX_WITH_KERNEL_USING_XMM
4906 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4907#else
4908 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4909#endif
4910}
4911
4912
4913/**
4914 * Reports world-switch error and dumps some useful debug info.
4915 *
4916 * @param pVM Pointer to the VM.
4917 * @param pVCpu Pointer to the VMCPU.
4918 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4919 * @param pCtx Pointer to the guest-CPU context.
4920 * @param pVmxTransient Pointer to the VMX transient structure (only
4921 * exitReason updated).
4922 */
4923static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4924{
4925 Assert(pVM);
4926 Assert(pVCpu);
4927 Assert(pCtx);
4928 Assert(pVmxTransient);
4929 HMVMX_ASSERT_PREEMPT_SAFE();
4930
4931 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4932 switch (rcVMRun)
4933 {
4934 case VERR_VMX_INVALID_VMXON_PTR:
4935 AssertFailed();
4936 break;
4937 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4938 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4939 {
4940 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4941 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4942 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4943 AssertRC(rc);
4944
4945 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4946 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4947 Cannot do it here as we may have been long preempted. */
4948
4949#ifdef VBOX_STRICT
4950 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4951 pVmxTransient->uExitReason));
4952 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4953 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4954 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4955 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4956 else
4957 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4958 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4959 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4960
4961 /* VMX control bits. */
4962 uint32_t u32Val;
4963 uint64_t u64Val;
4964 HMVMXHCUINTREG uHCReg;
4965 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4966 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4967 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4968 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4969 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4970 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4971 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4972 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4973 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4974 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4975 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4976 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4977 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4978 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4979 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4980 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4981 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4982 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4983 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4984 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4985 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4986 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4987 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4988 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4989 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4990 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4991 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4992 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4993 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4994 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4995 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4996 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4997 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4998 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4999 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5000 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5001 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5002 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5003 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5004 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5005 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5006 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5007
5008 /* Guest bits. */
5009 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5010 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5011 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5012 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5013 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5014 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5015 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
5016 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
5017
5018 /* Host bits. */
5019 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5020 Log4(("Host CR0 %#RHr\n", uHCReg));
5021 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5022 Log4(("Host CR3 %#RHr\n", uHCReg));
5023 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5024 Log4(("Host CR4 %#RHr\n", uHCReg));
5025
5026 RTGDTR HostGdtr;
5027 PCX86DESCHC pDesc;
5028 ASMGetGDTR(&HostGdtr);
5029 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
5030 Log4(("Host CS %#08x\n", u32Val));
5031 if (u32Val < HostGdtr.cbGdt)
5032 {
5033 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5034 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5035 }
5036
5037 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
5038 Log4(("Host DS %#08x\n", u32Val));
5039 if (u32Val < HostGdtr.cbGdt)
5040 {
5041 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5042 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5043 }
5044
5045 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
5046 Log4(("Host ES %#08x\n", u32Val));
5047 if (u32Val < HostGdtr.cbGdt)
5048 {
5049 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5050 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5051 }
5052
5053 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
5054 Log4(("Host FS %#08x\n", u32Val));
5055 if (u32Val < HostGdtr.cbGdt)
5056 {
5057 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5058 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5059 }
5060
5061 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
5062 Log4(("Host GS %#08x\n", u32Val));
5063 if (u32Val < HostGdtr.cbGdt)
5064 {
5065 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5066 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5067 }
5068
5069 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
5070 Log4(("Host SS %#08x\n", u32Val));
5071 if (u32Val < HostGdtr.cbGdt)
5072 {
5073 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5074 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5075 }
5076
5077 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
5078 Log4(("Host TR %#08x\n", u32Val));
5079 if (u32Val < HostGdtr.cbGdt)
5080 {
5081 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5082 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5083 }
5084
5085 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5086 Log4(("Host TR Base %#RHv\n", uHCReg));
5087 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5088 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5089 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5090 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5091 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5092 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5093 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5094 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5095 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5096 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5097 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5098 Log4(("Host RSP %#RHv\n", uHCReg));
5099 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5100 Log4(("Host RIP %#RHv\n", uHCReg));
5101# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5102 if (HMVMX_IS_64BIT_HOST_MODE())
5103 {
5104 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5105 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5106 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5107 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5108 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5109 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5110 }
5111# endif
5112#endif /* VBOX_STRICT */
5113 break;
5114 }
5115
5116 default:
5117 /* Impossible */
5118 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5119 break;
5120 }
5121 NOREF(pVM); NOREF(pCtx);
5122}
5123
5124
5125#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5126#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5127# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5128#endif
5129#ifdef VBOX_STRICT
5130static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5131{
5132 switch (idxField)
5133 {
5134 case VMX_VMCS_GUEST_RIP:
5135 case VMX_VMCS_GUEST_RSP:
5136 case VMX_VMCS_GUEST_SYSENTER_EIP:
5137 case VMX_VMCS_GUEST_SYSENTER_ESP:
5138 case VMX_VMCS_GUEST_GDTR_BASE:
5139 case VMX_VMCS_GUEST_IDTR_BASE:
5140 case VMX_VMCS_GUEST_CS_BASE:
5141 case VMX_VMCS_GUEST_DS_BASE:
5142 case VMX_VMCS_GUEST_ES_BASE:
5143 case VMX_VMCS_GUEST_FS_BASE:
5144 case VMX_VMCS_GUEST_GS_BASE:
5145 case VMX_VMCS_GUEST_SS_BASE:
5146 case VMX_VMCS_GUEST_LDTR_BASE:
5147 case VMX_VMCS_GUEST_TR_BASE:
5148 case VMX_VMCS_GUEST_CR3:
5149 return true;
5150 }
5151 return false;
5152}
5153
5154static bool hmR0VmxIsValidReadField(uint32_t idxField)
5155{
5156 switch (idxField)
5157 {
5158 /* Read-only fields. */
5159 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5160 return true;
5161 }
5162 /* Remaining readable fields should also be writable. */
5163 return hmR0VmxIsValidWriteField(idxField);
5164}
5165#endif /* VBOX_STRICT */
5166
5167
5168/**
5169 * Executes the specified handler in 64-bit mode.
5170 *
5171 * @returns VBox status code.
5172 * @param pVM Pointer to the VM.
5173 * @param pVCpu Pointer to the VMCPU.
5174 * @param pCtx Pointer to the guest CPU context.
5175 * @param enmOp The operation to perform.
5176 * @param cbParam Number of parameters.
5177 * @param paParam Array of 32-bit parameters.
5178 */
5179VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
5180 uint32_t *paParam)
5181{
5182 int rc, rc2;
5183 PHMGLOBALCPUINFO pCpu;
5184 RTHCPHYS HCPhysCpuPage;
5185 RTCCUINTREG uOldEflags;
5186
5187 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5188 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5189 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5190 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5191
5192#ifdef VBOX_STRICT
5193 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5194 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5195
5196 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5197 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5198#endif
5199
5200 /* Disable interrupts. */
5201 uOldEflags = ASMIntDisableFlags();
5202
5203#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5204 RTCPUID idHostCpu = RTMpCpuId();
5205 CPUMR0SetLApic(pVCpu, idHostCpu);
5206#endif
5207
5208 pCpu = HMR0GetCurrentCpu();
5209 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5210
5211 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5212 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5213
5214 /* Leave VMX Root Mode. */
5215 VMXDisable();
5216
5217 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5218
5219 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5220 CPUMSetHyperEIP(pVCpu, enmOp);
5221 for (int i = (int)cbParam - 1; i >= 0; i--)
5222 CPUMPushHyper(pVCpu, paParam[i]);
5223
5224 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5225
5226 /* Call the switcher. */
5227 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5228 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5229
5230 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5231 /* Make sure the VMX instructions don't cause #UD faults. */
5232 SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
5233
5234 /* Re-enter VMX Root Mode */
5235 rc2 = VMXEnable(HCPhysCpuPage);
5236 if (RT_FAILURE(rc2))
5237 {
5238 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5239 ASMSetFlags(uOldEflags);
5240 return rc2;
5241 }
5242
5243 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5244 AssertRC(rc2);
5245 Assert(!(ASMGetFlags() & X86_EFL_IF));
5246 ASMSetFlags(uOldEflags);
5247 return rc;
5248}
5249
5250
5251/**
5252 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5253 * supporting 64-bit guests.
5254 *
5255 * @returns VBox status code.
5256 * @param fResume Whether to VMLAUNCH or VMRESUME.
5257 * @param pCtx Pointer to the guest-CPU context.
5258 * @param pCache Pointer to the VMCS cache.
5259 * @param pVM Pointer to the VM.
5260 * @param pVCpu Pointer to the VMCPU.
5261 */
5262DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5263{
5264 uint32_t aParam[6];
5265 PHMGLOBALCPUINFO pCpu = NULL;
5266 RTHCPHYS HCPhysCpuPage = 0;
5267 int rc = VERR_INTERNAL_ERROR_5;
5268
5269 pCpu = HMR0GetCurrentCpu();
5270 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5271
5272#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5273 pCache->uPos = 1;
5274 pCache->interPD = PGMGetInterPaeCR3(pVM);
5275 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5276#endif
5277
5278#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5279 pCache->TestIn.HCPhysCpuPage = 0;
5280 pCache->TestIn.HCPhysVmcs = 0;
5281 pCache->TestIn.pCache = 0;
5282 pCache->TestOut.HCPhysVmcs = 0;
5283 pCache->TestOut.pCache = 0;
5284 pCache->TestOut.pCtx = 0;
5285 pCache->TestOut.eflags = 0;
5286#endif
5287
5288 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5289 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5290 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5291 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5292 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5293 aParam[5] = 0;
5294
5295#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5296 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5297 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5298#endif
5299 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
5300
5301#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5302 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5303 Assert(pCtx->dr[4] == 10);
5304 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5305#endif
5306
5307#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5308 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5309 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5310 pVCpu->hm.s.vmx.HCPhysVmcs));
5311 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5312 pCache->TestOut.HCPhysVmcs));
5313 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5314 pCache->TestOut.pCache));
5315 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5316 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5317 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5318 pCache->TestOut.pCtx));
5319 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5320#endif
5321 return rc;
5322}
5323
5324
5325/**
5326 * Initialize the VMCS-Read cache.
5327 *
5328 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5329 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5330 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5331 * (those that have a 32-bit FULL & HIGH part).
5332 *
5333 * @returns VBox status code.
5334 * @param pVM Pointer to the VM.
5335 * @param pVCpu Pointer to the VMCPU.
5336 */
5337static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5338{
5339#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5340{ \
5341 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5342 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5343 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5344 ++cReadFields; \
5345}
5346
5347 AssertPtr(pVM);
5348 AssertPtr(pVCpu);
5349 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5350 uint32_t cReadFields = 0;
5351
5352 /*
5353 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5354 * and serve to indicate exceptions to the rules.
5355 */
5356
5357 /* Guest-natural selector base fields. */
5358#if 0
5359 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5360 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5361 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5362#endif
5363 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5364 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5365 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5366 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5367 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5368 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5369 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5370 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5371 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5372 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5373 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5374 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5375#if 0
5376 /* Unused natural width guest-state fields. */
5377 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5378 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5379#endif
5380 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5381 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5382
5383 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5384#if 0
5385 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5386 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5387 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5388 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5389 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5390 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5391 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5392 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5393 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5394#endif
5395
5396 /* Natural width guest-state fields. */
5397 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5398#if 0
5399 /* Currently unused field. */
5400 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5401#endif
5402
5403 if (pVM->hm.s.fNestedPaging)
5404 {
5405 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5406 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5407 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5408 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5409 }
5410 else
5411 {
5412 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5413 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5414 }
5415
5416#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5417 return VINF_SUCCESS;
5418}
5419
5420
5421/**
5422 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5423 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5424 * darwin, running 64-bit guests).
5425 *
5426 * @returns VBox status code.
5427 * @param pVCpu Pointer to the VMCPU.
5428 * @param idxField The VMCS field encoding.
5429 * @param u64Val 16, 32 or 64-bit value.
5430 */
5431VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5432{
5433 int rc;
5434 switch (idxField)
5435 {
5436 /*
5437 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5438 */
5439 /* 64-bit Control fields. */
5440 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5441 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5442 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5443 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5444 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5445 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5446 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5447 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5448 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5449 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5450 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5451 case VMX_VMCS64_CTRL_EPTP_FULL:
5452 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5453 /* 64-bit Guest-state fields. */
5454 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5455 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5456 case VMX_VMCS64_GUEST_PAT_FULL:
5457 case VMX_VMCS64_GUEST_EFER_FULL:
5458 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5459 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5460 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5461 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5462 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5463 /* 64-bit Host-state fields. */
5464 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5465 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5466 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5467 {
5468 rc = VMXWriteVmcs32(idxField, u64Val);
5469 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5470 break;
5471 }
5472
5473 /*
5474 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5475 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5476 */
5477 /* Natural-width Guest-state fields. */
5478 case VMX_VMCS_GUEST_CR3:
5479 case VMX_VMCS_GUEST_ES_BASE:
5480 case VMX_VMCS_GUEST_CS_BASE:
5481 case VMX_VMCS_GUEST_SS_BASE:
5482 case VMX_VMCS_GUEST_DS_BASE:
5483 case VMX_VMCS_GUEST_FS_BASE:
5484 case VMX_VMCS_GUEST_GS_BASE:
5485 case VMX_VMCS_GUEST_LDTR_BASE:
5486 case VMX_VMCS_GUEST_TR_BASE:
5487 case VMX_VMCS_GUEST_GDTR_BASE:
5488 case VMX_VMCS_GUEST_IDTR_BASE:
5489 case VMX_VMCS_GUEST_RSP:
5490 case VMX_VMCS_GUEST_RIP:
5491 case VMX_VMCS_GUEST_SYSENTER_ESP:
5492 case VMX_VMCS_GUEST_SYSENTER_EIP:
5493 {
5494 if (!(u64Val >> 32))
5495 {
5496 /* If this field is 64-bit, VT-x will zero out the top bits. */
5497 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5498 }
5499 else
5500 {
5501 /* Assert that only the 32->64 switcher case should ever come here. */
5502 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5503 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5504 }
5505 break;
5506 }
5507
5508 default:
5509 {
5510 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5511 rc = VERR_INVALID_PARAMETER;
5512 break;
5513 }
5514 }
5515 AssertRCReturn(rc, rc);
5516 return rc;
5517}
5518
5519
5520/**
5521 * Queue up a VMWRITE by using the VMCS write cache.
5522 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5523 *
5524 * @param pVCpu Pointer to the VMCPU.
5525 * @param idxField The VMCS field encoding.
5526 * @param u64Val 16, 32 or 64-bit value.
5527 */
5528VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5529{
5530 AssertPtr(pVCpu);
5531 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5532
5533 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5534 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5535
5536 /* Make sure there are no duplicates. */
5537 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5538 {
5539 if (pCache->Write.aField[i] == idxField)
5540 {
5541 pCache->Write.aFieldVal[i] = u64Val;
5542 return VINF_SUCCESS;
5543 }
5544 }
5545
5546 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5547 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5548 pCache->Write.cValidEntries++;
5549 return VINF_SUCCESS;
5550}
5551
5552/* Enable later when the assembly code uses these as callbacks. */
5553#if 0
5554/*
5555 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5556 *
5557 * @param pVCpu Pointer to the VMCPU.
5558 * @param pCache Pointer to the VMCS cache.
5559 *
5560 * @remarks No-long-jump zone!!!
5561 */
5562VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5563{
5564 AssertPtr(pCache);
5565 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5566 {
5567 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5568 AssertRC(rc);
5569 }
5570 pCache->Write.cValidEntries = 0;
5571}
5572
5573
5574/**
5575 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
5576 *
5577 * @param pVCpu Pointer to the VMCPU.
5578 * @param pCache Pointer to the VMCS cache.
5579 *
5580 * @remarks No-long-jump zone!!!
5581 */
5582VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5583{
5584 AssertPtr(pCache);
5585 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5586 {
5587 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5588 AssertRC(rc);
5589 }
5590}
5591#endif
5592#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5593
5594
5595/**
5596 * Sets up the usage of TSC-offsetting and updates the VMCS.
5597 *
5598 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5599 * VMX preemption timer.
5600 *
5601 * @returns VBox status code.
5602 * @param pVM Pointer to the cross context VM structure.
5603 * @param pVCpu Pointer to the VMCPU.
5604 *
5605 * @remarks No-long-jump zone!!!
5606 */
5607static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5608{
5609 int rc;
5610 bool fOffsettedTsc;
5611 bool fParavirtTsc;
5612 if (pVM->hm.s.vmx.fUsePreemptTimer)
5613 {
5614 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5615 &fOffsettedTsc, &fParavirtTsc);
5616
5617 /* Make sure the returned values have sane upper and lower boundaries. */
5618 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5619 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5620 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5621 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5622
5623 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5624 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5625 }
5626 else
5627 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5628
5629 /** @todo later optimize this to be done elsewhere and not before every
5630 * VM-entry. */
5631 if (fParavirtTsc)
5632 {
5633 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5634 AssertRC(rc);
5635 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5636 }
5637
5638 if (fOffsettedTsc)
5639 {
5640 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5641 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5642
5643 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5644 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5645 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5646 }
5647 else
5648 {
5649 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5650 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5651 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5652 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5653 }
5654}
5655
5656
5657/**
5658 * Determines if an exception is a contributory exception.
5659 *
5660 * Contributory exceptions are ones which can cause double-faults unless the
5661 * original exception was a benign exception. Page-fault is intentionally not
5662 * included here as it's a conditional contributory exception.
5663 *
5664 * @returns true if the exception is contributory, false otherwise.
5665 * @param uVector The exception vector.
5666 */
5667DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5668{
5669 switch (uVector)
5670 {
5671 case X86_XCPT_GP:
5672 case X86_XCPT_SS:
5673 case X86_XCPT_NP:
5674 case X86_XCPT_TS:
5675 case X86_XCPT_DE:
5676 return true;
5677 default:
5678 break;
5679 }
5680 return false;
5681}
5682
5683
5684/**
5685 * Sets an event as a pending event to be injected into the guest.
5686 *
5687 * @param pVCpu Pointer to the VMCPU.
5688 * @param u32IntInfo The VM-entry interruption-information field.
5689 * @param cbInstr The VM-entry instruction length in bytes (for software
5690 * interrupts, exceptions and privileged software
5691 * exceptions).
5692 * @param u32ErrCode The VM-entry exception error code.
5693 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5694 * page-fault.
5695 *
5696 * @remarks Statistics counter assumes this is a guest event being injected or
5697 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5698 * always incremented.
5699 */
5700DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5701 RTGCUINTPTR GCPtrFaultAddress)
5702{
5703 Assert(!pVCpu->hm.s.Event.fPending);
5704 pVCpu->hm.s.Event.fPending = true;
5705 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5706 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5707 pVCpu->hm.s.Event.cbInstr = cbInstr;
5708 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5709
5710 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5711}
5712
5713
5714/**
5715 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5716 *
5717 * @param pVCpu Pointer to the VMCPU.
5718 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5719 * out-of-sync. Make sure to update the required fields
5720 * before using them.
5721 */
5722DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5723{
5724 NOREF(pMixedCtx);
5725 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5726 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5727 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5728 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5729}
5730
5731
5732/**
5733 * Handle a condition that occurred while delivering an event through the guest
5734 * IDT.
5735 *
5736 * @returns VBox status code (informational error codes included).
5737 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5738 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5739 * continue execution of the guest which will delivery the #DF.
5740 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5741 *
5742 * @param pVCpu Pointer to the VMCPU.
5743 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5744 * out-of-sync. Make sure to update the required fields
5745 * before using them.
5746 * @param pVmxTransient Pointer to the VMX transient structure.
5747 *
5748 * @remarks No-long-jump zone!!!
5749 */
5750static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5751{
5752 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5753
5754 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5755 AssertRCReturn(rc, rc);
5756 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5757 AssertRCReturn(rc, rc);
5758
5759 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5760 {
5761 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5762 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5763
5764 typedef enum
5765 {
5766 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5767 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5768 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5769 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5770 } VMXREFLECTXCPT;
5771
5772 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5773 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5774 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5775 {
5776 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5777 {
5778 enmReflect = VMXREFLECTXCPT_XCPT;
5779#ifdef VBOX_STRICT
5780 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5781 && uExitVector == X86_XCPT_PF)
5782 {
5783 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5784 }
5785#endif
5786 if ( uExitVector == X86_XCPT_PF
5787 && uIdtVector == X86_XCPT_PF)
5788 {
5789 pVmxTransient->fVectoringDoublePF = true;
5790 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5791 }
5792 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5793 && hmR0VmxIsContributoryXcpt(uExitVector)
5794 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5795 || uIdtVector == X86_XCPT_PF))
5796 {
5797 enmReflect = VMXREFLECTXCPT_DF;
5798 }
5799 else if (uIdtVector == X86_XCPT_DF)
5800 enmReflect = VMXREFLECTXCPT_TF;
5801 }
5802 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5803 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5804 {
5805 /*
5806 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5807 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5808 */
5809 enmReflect = VMXREFLECTXCPT_XCPT;
5810
5811 if (uExitVector == X86_XCPT_PF)
5812 {
5813 pVmxTransient->fVectoringPF = true;
5814 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5815 }
5816 }
5817 }
5818 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5819 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5820 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5821 {
5822 /*
5823 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5824 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5825 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5826 */
5827 enmReflect = VMXREFLECTXCPT_XCPT;
5828 }
5829
5830 /*
5831 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5832 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5833 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5834 *
5835 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5836 */
5837 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5838 && enmReflect == VMXREFLECTXCPT_XCPT
5839 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5840 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5841 {
5842 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5843 }
5844
5845 switch (enmReflect)
5846 {
5847 case VMXREFLECTXCPT_XCPT:
5848 {
5849 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5850 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5851 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5852
5853 uint32_t u32ErrCode = 0;
5854 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5855 {
5856 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5857 AssertRCReturn(rc, rc);
5858 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5859 }
5860
5861 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5862 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5863 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5864 rc = VINF_SUCCESS;
5865 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5866 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5867
5868 break;
5869 }
5870
5871 case VMXREFLECTXCPT_DF:
5872 {
5873 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5874 rc = VINF_HM_DOUBLE_FAULT;
5875 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5876 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5877
5878 break;
5879 }
5880
5881 case VMXREFLECTXCPT_TF:
5882 {
5883 rc = VINF_EM_RESET;
5884 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5885 uExitVector));
5886 break;
5887 }
5888
5889 default:
5890 Assert(rc == VINF_SUCCESS);
5891 break;
5892 }
5893 }
5894 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5895 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5896 && uExitVector != X86_XCPT_DF
5897 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5898 {
5899 /*
5900 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5901 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5902 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5903 */
5904 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5905 {
5906 Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
5907 pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5908 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5909 }
5910 }
5911
5912 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5913 return rc;
5914}
5915
5916
5917/**
5918 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5919 *
5920 * @returns VBox status code.
5921 * @param pVCpu Pointer to the VMCPU.
5922 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5923 * out-of-sync. Make sure to update the required fields
5924 * before using them.
5925 *
5926 * @remarks No-long-jump zone!!!
5927 */
5928static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5929{
5930 NOREF(pMixedCtx);
5931
5932 /*
5933 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5934 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5935 */
5936 VMMRZCallRing3Disable(pVCpu);
5937 HM_DISABLE_PREEMPT_IF_NEEDED();
5938
5939 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5940 {
5941 uint32_t uVal = 0;
5942 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5943 AssertRCReturn(rc, rc);
5944
5945 uint32_t uShadow = 0;
5946 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5947 AssertRCReturn(rc, rc);
5948
5949 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5950 CPUMSetGuestCR0(pVCpu, uVal);
5951 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5952 }
5953
5954 HM_RESTORE_PREEMPT_IF_NEEDED();
5955 VMMRZCallRing3Enable(pVCpu);
5956 return VINF_SUCCESS;
5957}
5958
5959
5960/**
5961 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5962 *
5963 * @returns VBox status code.
5964 * @param pVCpu Pointer to the VMCPU.
5965 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5966 * out-of-sync. Make sure to update the required fields
5967 * before using them.
5968 *
5969 * @remarks No-long-jump zone!!!
5970 */
5971static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5972{
5973 NOREF(pMixedCtx);
5974
5975 int rc = VINF_SUCCESS;
5976 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5977 {
5978 uint32_t uVal = 0;
5979 uint32_t uShadow = 0;
5980 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5981 AssertRCReturn(rc, rc);
5982 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5983 AssertRCReturn(rc, rc);
5984
5985 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5986 CPUMSetGuestCR4(pVCpu, uVal);
5987 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5988 }
5989 return rc;
5990}
5991
5992
5993/**
5994 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5995 *
5996 * @returns VBox status code.
5997 * @param pVCpu Pointer to the VMCPU.
5998 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5999 * out-of-sync. Make sure to update the required fields
6000 * before using them.
6001 *
6002 * @remarks No-long-jump zone!!!
6003 */
6004static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6005{
6006 int rc = VINF_SUCCESS;
6007 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6008 {
6009 uint64_t u64Val = 0;
6010 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6011 AssertRCReturn(rc, rc);
6012
6013 pMixedCtx->rip = u64Val;
6014 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6015 }
6016 return rc;
6017}
6018
6019
6020/**
6021 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6022 *
6023 * @returns VBox status code.
6024 * @param pVCpu Pointer to the VMCPU.
6025 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6026 * out-of-sync. Make sure to update the required fields
6027 * before using them.
6028 *
6029 * @remarks No-long-jump zone!!!
6030 */
6031static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6032{
6033 int rc = VINF_SUCCESS;
6034 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6035 {
6036 uint64_t u64Val = 0;
6037 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6038 AssertRCReturn(rc, rc);
6039
6040 pMixedCtx->rsp = u64Val;
6041 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6042 }
6043 return rc;
6044}
6045
6046
6047/**
6048 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6049 *
6050 * @returns VBox status code.
6051 * @param pVCpu Pointer to the VMCPU.
6052 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6053 * out-of-sync. Make sure to update the required fields
6054 * before using them.
6055 *
6056 * @remarks No-long-jump zone!!!
6057 */
6058static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6059{
6060 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6061 {
6062 uint32_t uVal = 0;
6063 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6064 AssertRCReturn(rc, rc);
6065
6066 pMixedCtx->eflags.u32 = uVal;
6067 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6068 {
6069 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6070 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6071
6072 pMixedCtx->eflags.Bits.u1VM = 0;
6073 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6074 }
6075
6076 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6077 }
6078 return VINF_SUCCESS;
6079}
6080
6081
6082/**
6083 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6084 * guest-CPU context.
6085 */
6086DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6087{
6088 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6089 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6090 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6091 return rc;
6092}
6093
6094
6095/**
6096 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6097 * from the guest-state area in the VMCS.
6098 *
6099 * @param pVCpu Pointer to the VMCPU.
6100 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6101 * out-of-sync. Make sure to update the required fields
6102 * before using them.
6103 *
6104 * @remarks No-long-jump zone!!!
6105 */
6106static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6107{
6108 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6109 {
6110 uint32_t uIntrState = 0;
6111 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6112 AssertRC(rc);
6113
6114 if (!uIntrState)
6115 {
6116 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6117 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6118
6119 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6120 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6121 }
6122 else
6123 {
6124 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6125 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6126 {
6127 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6128 AssertRC(rc);
6129 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6130 AssertRC(rc);
6131
6132 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6133 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6134 }
6135 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6136 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6137
6138 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6139 {
6140 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6141 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6142 }
6143 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6144 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6145 }
6146
6147 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6148 }
6149}
6150
6151
6152/**
6153 * Saves the guest's activity state.
6154 *
6155 * @returns VBox status code.
6156 * @param pVCpu Pointer to the VMCPU.
6157 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6158 * out-of-sync. Make sure to update the required fields
6159 * before using them.
6160 *
6161 * @remarks No-long-jump zone!!!
6162 */
6163static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6164{
6165 NOREF(pMixedCtx);
6166 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6167 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6168 return VINF_SUCCESS;
6169}
6170
6171
6172/**
6173 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6174 * the current VMCS into the guest-CPU context.
6175 *
6176 * @returns VBox status code.
6177 * @param pVCpu Pointer to the VMCPU.
6178 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6179 * out-of-sync. Make sure to update the required fields
6180 * before using them.
6181 *
6182 * @remarks No-long-jump zone!!!
6183 */
6184static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6185{
6186 int rc = VINF_SUCCESS;
6187 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6188 {
6189 uint32_t u32Val = 0;
6190 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6191 pMixedCtx->SysEnter.cs = u32Val;
6192 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6193 }
6194
6195 uint64_t u64Val = 0;
6196 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6197 {
6198 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6199 pMixedCtx->SysEnter.eip = u64Val;
6200 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6201 }
6202 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6203 {
6204 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6205 pMixedCtx->SysEnter.esp = u64Val;
6206 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6207 }
6208 return rc;
6209}
6210
6211
6212/**
6213 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6214 * the CPU back into the guest-CPU context.
6215 *
6216 * @returns VBox status code.
6217 * @param pVCpu Pointer to the VMCPU.
6218 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6219 * out-of-sync. Make sure to update the required fields
6220 * before using them.
6221 *
6222 * @remarks No-long-jump zone!!!
6223 */
6224static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6225{
6226#if HC_ARCH_BITS == 64
6227 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
6228 {
6229 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6230 VMMRZCallRing3Disable(pVCpu);
6231 HM_DISABLE_PREEMPT_IF_NEEDED();
6232
6233 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6234 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6235 {
6236 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6237 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6238 }
6239
6240 HM_RESTORE_PREEMPT_IF_NEEDED();
6241 VMMRZCallRing3Enable(pVCpu);
6242 }
6243 else
6244 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6245#else
6246 NOREF(pMixedCtx);
6247 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6248#endif
6249
6250 return VINF_SUCCESS;
6251}
6252
6253
6254/**
6255 * Saves the auto load/store'd guest MSRs from the current VMCS into
6256 * the guest-CPU context.
6257 *
6258 * @returns VBox status code.
6259 * @param pVCpu Pointer to the VMCPU.
6260 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6261 * out-of-sync. Make sure to update the required fields
6262 * before using them.
6263 *
6264 * @remarks No-long-jump zone!!!
6265 */
6266static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6267{
6268 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6269 return VINF_SUCCESS;
6270
6271 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6272 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6273 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6274 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6275 {
6276 switch (pMsr->u32Msr)
6277 {
6278 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6279 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6280 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6281 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6282 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6283 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6284 break;
6285
6286 default:
6287 {
6288 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6289 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6290 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6291 }
6292 }
6293 }
6294
6295 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6296 return VINF_SUCCESS;
6297}
6298
6299
6300/**
6301 * Saves the guest control registers from the current VMCS into the guest-CPU
6302 * context.
6303 *
6304 * @returns VBox status code.
6305 * @param pVCpu Pointer to the VMCPU.
6306 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6307 * out-of-sync. Make sure to update the required fields
6308 * before using them.
6309 *
6310 * @remarks No-long-jump zone!!!
6311 */
6312static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6313{
6314 /* Guest CR0. Guest FPU. */
6315 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6316 AssertRCReturn(rc, rc);
6317
6318 /* Guest CR4. */
6319 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6320 AssertRCReturn(rc, rc);
6321
6322 /* Guest CR2 - updated always during the world-switch or in #PF. */
6323 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6324 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6325 {
6326 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6327 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6328
6329 PVM pVM = pVCpu->CTX_SUFF(pVM);
6330 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6331 || ( pVM->hm.s.fNestedPaging
6332 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6333 {
6334 uint64_t u64Val = 0;
6335 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6336 if (pMixedCtx->cr3 != u64Val)
6337 {
6338 CPUMSetGuestCR3(pVCpu, u64Val);
6339 if (VMMRZCallRing3IsEnabled(pVCpu))
6340 {
6341 PGMUpdateCR3(pVCpu, u64Val);
6342 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6343 }
6344 else
6345 {
6346 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6347 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6348 }
6349 }
6350
6351 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6352 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6353 {
6354 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
6355 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
6356 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
6357 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
6358
6359 if (VMMRZCallRing3IsEnabled(pVCpu))
6360 {
6361 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6362 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6363 }
6364 else
6365 {
6366 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6367 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6368 }
6369 }
6370 }
6371
6372 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6373 }
6374
6375 /*
6376 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6377 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6378 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6379 *
6380 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6381 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6382 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6383 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6384 *
6385 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6386 */
6387 if (VMMRZCallRing3IsEnabled(pVCpu))
6388 {
6389 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6390 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6391
6392 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6393 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6394
6395 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6396 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6397 }
6398
6399 return rc;
6400}
6401
6402
6403/**
6404 * Reads a guest segment register from the current VMCS into the guest-CPU
6405 * context.
6406 *
6407 * @returns VBox status code.
6408 * @param pVCpu Pointer to the VMCPU.
6409 * @param idxSel Index of the selector in the VMCS.
6410 * @param idxLimit Index of the segment limit in the VMCS.
6411 * @param idxBase Index of the segment base in the VMCS.
6412 * @param idxAccess Index of the access rights of the segment in the VMCS.
6413 * @param pSelReg Pointer to the segment selector.
6414 *
6415 * @remarks No-long-jump zone!!!
6416 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6417 * macro as that takes care of whether to read from the VMCS cache or
6418 * not.
6419 */
6420DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6421 PCPUMSELREG pSelReg)
6422{
6423 NOREF(pVCpu);
6424
6425 uint32_t u32Val = 0;
6426 int rc = VMXReadVmcs32(idxSel, &u32Val);
6427 AssertRCReturn(rc, rc);
6428 pSelReg->Sel = (uint16_t)u32Val;
6429 pSelReg->ValidSel = (uint16_t)u32Val;
6430 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6431
6432 rc = VMXReadVmcs32(idxLimit, &u32Val);
6433 AssertRCReturn(rc, rc);
6434 pSelReg->u32Limit = u32Val;
6435
6436 uint64_t u64Val = 0;
6437 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6438 AssertRCReturn(rc, rc);
6439 pSelReg->u64Base = u64Val;
6440
6441 rc = VMXReadVmcs32(idxAccess, &u32Val);
6442 AssertRCReturn(rc, rc);
6443 pSelReg->Attr.u = u32Val;
6444
6445 /*
6446 * If VT-x marks the segment as unusable, most other bits remain undefined:
6447 * - For CS the L, D and G bits have meaning.
6448 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6449 * - For the remaining data segments no bits are defined.
6450 *
6451 * The present bit and the unusable bit has been observed to be set at the
6452 * same time (the selector was supposed to be invalid as we started executing
6453 * a V8086 interrupt in ring-0).
6454 *
6455 * What should be important for the rest of the VBox code, is that the P bit is
6456 * cleared. Some of the other VBox code recognizes the unusable bit, but
6457 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6458 * safe side here, we'll strip off P and other bits we don't care about. If
6459 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6460 *
6461 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6462 */
6463 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6464 {
6465 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6466
6467 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6468 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6469 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6470
6471 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6472#ifdef DEBUG_bird
6473 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6474 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6475 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6476#endif
6477 }
6478 return VINF_SUCCESS;
6479}
6480
6481
6482#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6483# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6484 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6485 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6486#else
6487# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6488 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6489 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6490#endif
6491
6492
6493/**
6494 * Saves the guest segment registers from the current VMCS into the guest-CPU
6495 * context.
6496 *
6497 * @returns VBox status code.
6498 * @param pVCpu Pointer to the VMCPU.
6499 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6500 * out-of-sync. Make sure to update the required fields
6501 * before using them.
6502 *
6503 * @remarks No-long-jump zone!!!
6504 */
6505static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6506{
6507 /* Guest segment registers. */
6508 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6509 {
6510 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6511 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6512 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6513 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6514 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6515 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6516 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6517
6518 /* Restore segment attributes for real-on-v86 mode hack. */
6519 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6520 {
6521 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6522 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6523 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6524 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6525 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6526 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6527 }
6528 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6529 }
6530
6531 return VINF_SUCCESS;
6532}
6533
6534
6535/**
6536 * Saves the guest descriptor table registers and task register from the current
6537 * VMCS into the guest-CPU context.
6538 *
6539 * @returns VBox status code.
6540 * @param pVCpu Pointer to the VMCPU.
6541 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6542 * out-of-sync. Make sure to update the required fields
6543 * before using them.
6544 *
6545 * @remarks No-long-jump zone!!!
6546 */
6547static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6548{
6549 int rc = VINF_SUCCESS;
6550
6551 /* Guest LDTR. */
6552 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6553 {
6554 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6555 AssertRCReturn(rc, rc);
6556 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6557 }
6558
6559 /* Guest GDTR. */
6560 uint64_t u64Val = 0;
6561 uint32_t u32Val = 0;
6562 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6563 {
6564 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6565 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6566 pMixedCtx->gdtr.pGdt = u64Val;
6567 pMixedCtx->gdtr.cbGdt = u32Val;
6568 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6569 }
6570
6571 /* Guest IDTR. */
6572 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6573 {
6574 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6575 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6576 pMixedCtx->idtr.pIdt = u64Val;
6577 pMixedCtx->idtr.cbIdt = u32Val;
6578 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6579 }
6580
6581 /* Guest TR. */
6582 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6583 {
6584 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6585 AssertRCReturn(rc, rc);
6586
6587 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6588 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6589 {
6590 rc = VMXLOCAL_READ_SEG(TR, tr);
6591 AssertRCReturn(rc, rc);
6592 }
6593 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6594 }
6595 return rc;
6596}
6597
6598#undef VMXLOCAL_READ_SEG
6599
6600
6601/**
6602 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6603 * context.
6604 *
6605 * @returns VBox status code.
6606 * @param pVCpu Pointer to the VMCPU.
6607 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6608 * out-of-sync. Make sure to update the required fields
6609 * before using them.
6610 *
6611 * @remarks No-long-jump zone!!!
6612 */
6613static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6614{
6615 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6616 {
6617 if (!pVCpu->hm.s.fUsingHyperDR7)
6618 {
6619 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6620 uint32_t u32Val;
6621 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6622 pMixedCtx->dr[7] = u32Val;
6623 }
6624
6625 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6626 }
6627 return VINF_SUCCESS;
6628}
6629
6630
6631/**
6632 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6633 *
6634 * @returns VBox status code.
6635 * @param pVCpu Pointer to the VMCPU.
6636 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6637 * out-of-sync. Make sure to update the required fields
6638 * before using them.
6639 *
6640 * @remarks No-long-jump zone!!!
6641 */
6642static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6643{
6644 NOREF(pMixedCtx);
6645
6646 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6647 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6648 return VINF_SUCCESS;
6649}
6650
6651
6652/**
6653 * Saves the entire guest state from the currently active VMCS into the
6654 * guest-CPU context. This essentially VMREADs all guest-data.
6655 *
6656 * @returns VBox status code.
6657 * @param pVCpu Pointer to the VMCPU.
6658 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6659 * out-of-sync. Make sure to update the required fields
6660 * before using them.
6661 */
6662static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6663{
6664 Assert(pVCpu);
6665 Assert(pMixedCtx);
6666
6667 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6668 return VINF_SUCCESS;
6669
6670 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6671 again on the ring-3 callback path, there is no real need to. */
6672 if (VMMRZCallRing3IsEnabled(pVCpu))
6673 VMMR0LogFlushDisable(pVCpu);
6674 else
6675 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6676 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6677
6678 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6679 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6680
6681 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6682 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6683
6684 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6685 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6686
6687 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6688 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6689
6690 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6691 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6692
6693 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6694 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6695
6696 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6697 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6698
6699 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6700 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6701
6702 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6703 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6704
6705 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6706 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6707
6708 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6709 ("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6710
6711 if (VMMRZCallRing3IsEnabled(pVCpu))
6712 VMMR0LogFlushEnable(pVCpu);
6713
6714 return rc;
6715}
6716
6717
6718/**
6719 * Check per-VM and per-VCPU force flag actions that require us to go back to
6720 * ring-3 for one reason or another.
6721 *
6722 * @returns VBox status code (information status code included).
6723 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6724 * ring-3.
6725 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6726 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6727 * interrupts)
6728 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6729 * all EMTs to be in ring-3.
6730 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6731 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6732 * to the EM loop.
6733 *
6734 * @param pVM Pointer to the VM.
6735 * @param pVCpu Pointer to the VMCPU.
6736 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6737 * out-of-sync. Make sure to update the required fields
6738 * before using them.
6739 */
6740static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6741{
6742 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6743
6744 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6745 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6746 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6747 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6748 {
6749 /* We need the control registers now, make sure the guest-CPU context is updated. */
6750 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6751 AssertRCReturn(rc3, rc3);
6752
6753 /* Pending HM CR3 sync. */
6754 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6755 {
6756 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6757 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6758 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6759 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6760 }
6761
6762 /* Pending HM PAE PDPEs. */
6763 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6764 {
6765 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6766 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6767 }
6768
6769 /* Pending PGM C3 sync. */
6770 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6771 {
6772 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6773 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6774 if (rc2 != VINF_SUCCESS)
6775 {
6776 AssertRC(rc2);
6777 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6778 return rc2;
6779 }
6780 }
6781
6782 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6783 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6784 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6785 {
6786 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6787 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6788 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6789 return rc2;
6790 }
6791
6792 /* Pending VM request packets, such as hardware interrupts. */
6793 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6794 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6795 {
6796 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6797 return VINF_EM_PENDING_REQUEST;
6798 }
6799
6800 /* Pending PGM pool flushes. */
6801 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6802 {
6803 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6804 return VINF_PGM_POOL_FLUSH_PENDING;
6805 }
6806
6807 /* Pending DMA requests. */
6808 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6809 {
6810 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6811 return VINF_EM_RAW_TO_R3;
6812 }
6813 }
6814
6815 return VINF_SUCCESS;
6816}
6817
6818
6819/**
6820 * Converts any TRPM trap into a pending HM event. This is typically used when
6821 * entering from ring-3 (not longjmp returns).
6822 *
6823 * @param pVCpu Pointer to the VMCPU.
6824 */
6825static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6826{
6827 Assert(TRPMHasTrap(pVCpu));
6828 Assert(!pVCpu->hm.s.Event.fPending);
6829
6830 uint8_t uVector;
6831 TRPMEVENT enmTrpmEvent;
6832 RTGCUINT uErrCode;
6833 RTGCUINTPTR GCPtrFaultAddress;
6834 uint8_t cbInstr;
6835
6836 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6837 AssertRC(rc);
6838
6839 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6840 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6841 if (enmTrpmEvent == TRPM_TRAP)
6842 {
6843 switch (uVector)
6844 {
6845 case X86_XCPT_NMI:
6846 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6847 break;
6848
6849 case X86_XCPT_BP:
6850 case X86_XCPT_OF:
6851 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6852 break;
6853
6854 case X86_XCPT_PF:
6855 case X86_XCPT_DF:
6856 case X86_XCPT_TS:
6857 case X86_XCPT_NP:
6858 case X86_XCPT_SS:
6859 case X86_XCPT_GP:
6860 case X86_XCPT_AC:
6861 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6862 /* no break! */
6863 default:
6864 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6865 break;
6866 }
6867 }
6868 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6869 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6870 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6871 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6872 else
6873 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6874
6875 rc = TRPMResetTrap(pVCpu);
6876 AssertRC(rc);
6877 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6878 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6879
6880 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6881 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6882}
6883
6884
6885/**
6886 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6887 * VT-x to execute any instruction.
6888 *
6889 * @param pvCpu Pointer to the VMCPU.
6890 */
6891static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6892{
6893 Assert(pVCpu->hm.s.Event.fPending);
6894
6895 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6896 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6897 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6898 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6899
6900 /* If a trap was already pending, we did something wrong! */
6901 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6902
6903 TRPMEVENT enmTrapType;
6904 switch (uVectorType)
6905 {
6906 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6907 enmTrapType = TRPM_HARDWARE_INT;
6908 break;
6909
6910 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6911 enmTrapType = TRPM_SOFTWARE_INT;
6912 break;
6913
6914 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6915 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6916 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6917 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6918 enmTrapType = TRPM_TRAP;
6919 break;
6920
6921 default:
6922 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6923 enmTrapType = TRPM_32BIT_HACK;
6924 break;
6925 }
6926
6927 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6928
6929 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6930 AssertRC(rc);
6931
6932 if (fErrorCodeValid)
6933 TRPMSetErrorCode(pVCpu, uErrorCode);
6934
6935 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6936 && uVector == X86_XCPT_PF)
6937 {
6938 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6939 }
6940 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6941 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6942 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6943 {
6944 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6945 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6946 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6947 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6948 }
6949 pVCpu->hm.s.Event.fPending = false;
6950}
6951
6952
6953/**
6954 * Does the necessary state syncing before returning to ring-3 for any reason
6955 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6956 *
6957 * @returns VBox status code.
6958 * @param pVM Pointer to the VM.
6959 * @param pVCpu Pointer to the VMCPU.
6960 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6961 * be out-of-sync. Make sure to update the required
6962 * fields before using them.
6963 * @param fSaveGuestState Whether to save the guest state or not.
6964 *
6965 * @remarks No-long-jmp zone!!!
6966 */
6967static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6968{
6969 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6970 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6971
6972 RTCPUID idCpu = RTMpCpuId();
6973 Log4Func(("HostCpuId=%u\n", idCpu));
6974
6975 /*
6976 * !!! IMPORTANT !!!
6977 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6978 */
6979
6980 /* Save the guest state if necessary. */
6981 if ( fSaveGuestState
6982 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6983 {
6984 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6985 AssertRCReturn(rc, rc);
6986 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6987 }
6988
6989 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6990 if (CPUMIsGuestFPUStateActive(pVCpu))
6991 {
6992 /* We shouldn't reload CR0 without saving it first. */
6993 if (!fSaveGuestState)
6994 {
6995 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6996 AssertRCReturn(rc, rc);
6997 }
6998 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6999 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
7000 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7001 }
7002
7003 /* Restore host debug registers if necessary and resync on next R0 reentry. */
7004#ifdef VBOX_STRICT
7005 if (CPUMIsHyperDebugStateActive(pVCpu))
7006 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7007#endif
7008 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7009 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7010 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7011 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7012
7013#if HC_ARCH_BITS == 64
7014 /* Restore host-state bits that VT-x only restores partially. */
7015 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7016 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7017 {
7018 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7019 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7020 }
7021 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7022#endif
7023
7024#if HC_ARCH_BITS == 64
7025 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7026 if ( pVM->hm.s.fAllow64BitGuests
7027 && pVCpu->hm.s.vmx.fLazyMsrs)
7028 {
7029 /* We shouldn't reload the guest MSRs without saving it first. */
7030 if (!fSaveGuestState)
7031 {
7032 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7033 AssertRCReturn(rc, rc);
7034 }
7035 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7036 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7037 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7038 }
7039#endif
7040
7041 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7042 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7043
7044 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7045 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7046 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7047 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7048 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7049 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7050 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7051 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7052
7053 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7054
7055 /** @todo This partially defeats the purpose of having preemption hooks.
7056 * The problem is, deregistering the hooks should be moved to a place that
7057 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7058 * context.
7059 */
7060 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7061 {
7062 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7063 AssertRCReturn(rc, rc);
7064
7065 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7066 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7067 }
7068 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7069 NOREF(idCpu);
7070
7071 return VINF_SUCCESS;
7072}
7073
7074
7075/**
7076 * Leaves the VT-x session.
7077 *
7078 * @returns VBox status code.
7079 * @param pVM Pointer to the VM.
7080 * @param pVCpu Pointer to the VMCPU.
7081 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7082 * out-of-sync. Make sure to update the required fields
7083 * before using them.
7084 *
7085 * @remarks No-long-jmp zone!!!
7086 */
7087DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7088{
7089 HM_DISABLE_PREEMPT_IF_NEEDED();
7090 HMVMX_ASSERT_CPU_SAFE();
7091 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7092 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7093
7094 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7095 and done this from the VMXR0ThreadCtxCallback(). */
7096 if (!pVCpu->hm.s.fLeaveDone)
7097 {
7098 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7099 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
7100 pVCpu->hm.s.fLeaveDone = true;
7101 }
7102 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7103
7104 /*
7105 * !!! IMPORTANT !!!
7106 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7107 */
7108
7109 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7110 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
7111 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7112 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7113 VMMR0ThreadCtxHooksDeregister(pVCpu);
7114
7115 /* Leave HM context. This takes care of local init (term). */
7116 int rc = HMR0LeaveCpu(pVCpu);
7117
7118 HM_RESTORE_PREEMPT_IF_NEEDED();
7119
7120 return rc;
7121}
7122
7123
7124/**
7125 * Does the necessary state syncing before doing a longjmp to ring-3.
7126 *
7127 * @returns VBox status code.
7128 * @param pVM Pointer to the VM.
7129 * @param pVCpu Pointer to the VMCPU.
7130 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7131 * out-of-sync. Make sure to update the required fields
7132 * before using them.
7133 *
7134 * @remarks No-long-jmp zone!!!
7135 */
7136DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7137{
7138 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7139}
7140
7141
7142/**
7143 * Take necessary actions before going back to ring-3.
7144 *
7145 * An action requires us to go back to ring-3. This function does the necessary
7146 * steps before we can safely return to ring-3. This is not the same as longjmps
7147 * to ring-3, this is voluntary and prepares the guest so it may continue
7148 * executing outside HM (recompiler/IEM).
7149 *
7150 * @returns VBox status code.
7151 * @param pVM Pointer to the VM.
7152 * @param pVCpu Pointer to the VMCPU.
7153 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7154 * out-of-sync. Make sure to update the required fields
7155 * before using them.
7156 * @param rcExit The reason for exiting to ring-3. Can be
7157 * VINF_VMM_UNKNOWN_RING3_CALL.
7158 */
7159static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
7160{
7161 Assert(pVM);
7162 Assert(pVCpu);
7163 Assert(pMixedCtx);
7164 HMVMX_ASSERT_PREEMPT_SAFE();
7165
7166 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7167 {
7168 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7169 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7170 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7171 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7172 }
7173
7174 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7175 VMMRZCallRing3Disable(pVCpu);
7176 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
7177
7178 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7179 if (pVCpu->hm.s.Event.fPending)
7180 {
7181 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7182 Assert(!pVCpu->hm.s.Event.fPending);
7183 }
7184
7185 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7186 and if we're injecting an event we should have a TRPM trap pending. */
7187 Assert(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu));
7188 Assert(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu));
7189
7190 /* Save guest state and restore host state bits. */
7191 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7192 AssertRCReturn(rc, rc);
7193 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7194 /* Thread-context hooks are unregistered at this point!!! */
7195
7196 /* Sync recompiler state. */
7197 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7198 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7199 | CPUM_CHANGED_LDTR
7200 | CPUM_CHANGED_GDTR
7201 | CPUM_CHANGED_IDTR
7202 | CPUM_CHANGED_TR
7203 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7204 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7205 if ( pVM->hm.s.fNestedPaging
7206 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7207 {
7208 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7209 }
7210
7211 Assert(!pVCpu->hm.s.fClearTrapFlag);
7212
7213 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7214 if (rcExit != VINF_EM_RAW_INTERRUPT)
7215 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7216
7217 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7218
7219 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7220 VMMRZCallRing3RemoveNotification(pVCpu);
7221 VMMRZCallRing3Enable(pVCpu);
7222
7223 return rc;
7224}
7225
7226
7227/**
7228 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7229 * longjump to ring-3 and possibly get preempted.
7230 *
7231 * @returns VBox status code.
7232 * @param pVCpu Pointer to the VMCPU.
7233 * @param enmOperation The operation causing the ring-3 longjump.
7234 * @param pvUser Opaque pointer to the guest-CPU context. The data
7235 * may be out-of-sync. Make sure to update the required
7236 * fields before using them.
7237 */
7238DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7239{
7240 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7241 {
7242 /*
7243 * !!! IMPORTANT !!!
7244 * If you modify code here, make sure to check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs
7245 * to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
7246 */
7247 VMMRZCallRing3RemoveNotification(pVCpu);
7248 VMMRZCallRing3Disable(pVCpu);
7249 HM_DISABLE_PREEMPT_IF_NEEDED();
7250
7251 PVM pVM = pVCpu->CTX_SUFF(pVM);
7252 if (CPUMIsGuestFPUStateActive(pVCpu))
7253 CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
7254
7255 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7256
7257#if HC_ARCH_BITS == 64
7258 /* Restore host-state bits that VT-x only restores partially. */
7259 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7260 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7261 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7262 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7263
7264 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7265 if ( pVM->hm.s.fAllow64BitGuests
7266 && pVCpu->hm.s.vmx.fLazyMsrs)
7267 {
7268 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7269 }
7270#endif
7271 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7272 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7273 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7274 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7275 {
7276 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7277 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7278 }
7279
7280 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7281 VMMR0ThreadCtxHooksDeregister(pVCpu);
7282
7283 HMR0LeaveCpu(pVCpu);
7284 HM_RESTORE_PREEMPT_IF_NEEDED();
7285 return VINF_SUCCESS;
7286 }
7287
7288 Assert(pVCpu);
7289 Assert(pvUser);
7290 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7291 HMVMX_ASSERT_PREEMPT_SAFE();
7292
7293 VMMRZCallRing3Disable(pVCpu);
7294 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7295
7296 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7297 enmOperation));
7298
7299 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7300 AssertRCReturn(rc, rc);
7301
7302 VMMRZCallRing3Enable(pVCpu);
7303 return VINF_SUCCESS;
7304}
7305
7306
7307/**
7308 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7309 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7310 *
7311 * @param pVCpu Pointer to the VMCPU.
7312 */
7313DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7314{
7315 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7316 {
7317 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7318 {
7319 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7320 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7321 AssertRC(rc);
7322 Log4(("Setup interrupt-window exiting\n"));
7323 }
7324 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7325}
7326
7327
7328/**
7329 * Clears the interrupt-window exiting control in the VMCS.
7330 *
7331 * @param pVCpu Pointer to the VMCPU.
7332 */
7333DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7334{
7335 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7336 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7337 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7338 AssertRC(rc);
7339 Log4(("Cleared interrupt-window exiting\n"));
7340}
7341
7342
7343/**
7344 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7345 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7346 *
7347 * @param pVCpu Pointer to the VMCPU.
7348 */
7349DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7350{
7351 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7352 {
7353 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7354 {
7355 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7356 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7357 AssertRC(rc);
7358 Log4(("Setup NMI-window exiting\n"));
7359 }
7360 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7361}
7362
7363
7364/**
7365 * Clears the NMI-window exiting control in the VMCS.
7366 *
7367 * @param pVCpu Pointer to the VMCPU.
7368 */
7369DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7370{
7371 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7372 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7373 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7374 AssertRC(rc);
7375 Log4(("Cleared NMI-window exiting\n"));
7376}
7377
7378
7379/**
7380 * Evaluates the event to be delivered to the guest and sets it as the pending
7381 * event.
7382 *
7383 * @param pVCpu Pointer to the VMCPU.
7384 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7385 * out-of-sync. Make sure to update the required fields
7386 * before using them.
7387 */
7388static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7389{
7390 Assert(!pVCpu->hm.s.Event.fPending);
7391
7392 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7393 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7394 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7395 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7396 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7397
7398 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7399 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7400 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7401 Assert(!TRPMHasTrap(pVCpu));
7402
7403 /*
7404 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7405 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7406 */
7407 /** @todo SMI. SMIs take priority over NMIs. */
7408 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7409 {
7410 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7411 if ( !fBlockNmi
7412 && !fBlockSti
7413 && !fBlockMovSS)
7414 {
7415 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7416 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7417 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7418
7419 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7420 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7421 }
7422 else
7423 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7424 }
7425 /*
7426 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
7427 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt.
7428 */
7429 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7430 && !pVCpu->hm.s.fSingleInstruction)
7431 {
7432 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7433 AssertRC(rc);
7434 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7435 if ( !fBlockInt
7436 && !fBlockSti
7437 && !fBlockMovSS)
7438 {
7439 uint8_t u8Interrupt;
7440 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7441 if (RT_SUCCESS(rc))
7442 {
7443 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7444 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7445 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7446
7447 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7448 }
7449 else
7450 {
7451 /** @todo Does this actually happen? If not turn it into an assertion. */
7452 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
7453 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7454 }
7455 }
7456 else
7457 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7458 }
7459}
7460
7461
7462/**
7463 * Sets a pending-debug exception to be delivered to the guest if the guest is
7464 * single-stepping.
7465 *
7466 * @param pVCpu Pointer to the VMCPU.
7467 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7468 * out-of-sync. Make sure to update the required fields
7469 * before using them.
7470 */
7471DECLINLINE(void) hmR0VmxSetPendingDebugXcpt(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7472{
7473 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7474 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7475 {
7476 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7477 AssertRC(rc);
7478 }
7479}
7480
7481
7482/**
7483 * Injects any pending events into the guest if the guest is in a state to
7484 * receive them.
7485 *
7486 * @returns VBox status code (informational status codes included).
7487 * @param pVCpu Pointer to the VMCPU.
7488 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7489 * out-of-sync. Make sure to update the required fields
7490 * before using them.
7491 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7492 * return VINF_EM_DBG_STEPPED if the event was
7493 * dispatched directly.
7494 */
7495static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
7496{
7497 HMVMX_ASSERT_PREEMPT_SAFE();
7498 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7499
7500 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7501 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7502 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7503 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7504
7505 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7506 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7507 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7508 Assert(!TRPMHasTrap(pVCpu));
7509
7510 int rc = VINF_SUCCESS;
7511 if (pVCpu->hm.s.Event.fPending)
7512 {
7513 /*
7514 * Clear any interrupt-window exiting control if we're going to inject an interrupt. Saves one extra
7515 * VM-exit in situations where we previously setup interrupt-window exiting but got other VM-exits and
7516 * ended up enabling interrupts outside VT-x.
7517 */
7518 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7519 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7520 && uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7521 {
7522 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7523 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7524 }
7525
7526#ifdef VBOX_STRICT
7527 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7528 {
7529 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7530 Assert(!fBlockInt);
7531 Assert(!fBlockSti);
7532 Assert(!fBlockMovSS);
7533 }
7534 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7535 {
7536 bool fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7537 Assert(!fBlockSti);
7538 Assert(!fBlockMovSS);
7539 Assert(!fBlockNmi);
7540 }
7541#endif
7542 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7543 (uint8_t)uIntType));
7544 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7545 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping, &uIntrState);
7546 AssertRCReturn(rc, rc);
7547
7548 /* Update the interruptibility-state as it could have been changed by
7549 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7550 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7551 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7552
7553#ifdef VBOX_WITH_STATISTICS
7554 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7555 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7556 else
7557 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7558#endif
7559 }
7560
7561 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7562 if ( fBlockSti
7563 || fBlockMovSS)
7564 {
7565 if ( !pVCpu->hm.s.fSingleInstruction
7566 && !DBGFIsStepping(pVCpu))
7567 {
7568 /*
7569 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7570 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7571 * See Intel spec. 27.3.4 "Saving Non-Register State".
7572 */
7573 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7574 AssertRCReturn(rc2, rc2);
7575 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
7576 }
7577 else if (pMixedCtx->eflags.Bits.u1TF)
7578 {
7579 /*
7580 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7581 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7582 */
7583 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7584 uIntrState = 0;
7585 }
7586 }
7587
7588 /*
7589 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7590 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7591 */
7592 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7593 AssertRC(rc2);
7594
7595 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
7596 NOREF(fBlockMovSS); NOREF(fBlockSti);
7597 return rc;
7598}
7599
7600
7601/**
7602 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7603 *
7604 * @param pVCpu Pointer to the VMCPU.
7605 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7606 * out-of-sync. Make sure to update the required fields
7607 * before using them.
7608 */
7609DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7610{
7611 NOREF(pMixedCtx);
7612 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7613 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7614}
7615
7616
7617/**
7618 * Injects a double-fault (#DF) exception into the VM.
7619 *
7620 * @returns VBox status code (informational status code included).
7621 * @param pVCpu Pointer to the VMCPU.
7622 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7623 * out-of-sync. Make sure to update the required fields
7624 * before using them.
7625 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7626 * and should return VINF_EM_DBG_STEPPED if the event
7627 * is injected directly (register modified by us, not
7628 * by hardware on VM-entry).
7629 * @param puIntrState Pointer to the current guest interruptibility-state.
7630 * This interruptibility-state will be updated if
7631 * necessary. This cannot not be NULL.
7632 */
7633DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7634{
7635 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7636 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7637 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7638 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7639 fStepping, puIntrState);
7640}
7641
7642
7643/**
7644 * Sets a debug (#DB) exception as pending-for-injection into the VM.
7645 *
7646 * @param pVCpu Pointer to the VMCPU.
7647 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7648 * out-of-sync. Make sure to update the required fields
7649 * before using them.
7650 */
7651DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7652{
7653 NOREF(pMixedCtx);
7654 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7655 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7656 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7657}
7658
7659
7660/**
7661 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
7662 *
7663 * @param pVCpu Pointer to the VMCPU.
7664 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7665 * out-of-sync. Make sure to update the required fields
7666 * before using them.
7667 * @param cbInstr The value of RIP that is to be pushed on the guest
7668 * stack.
7669 */
7670DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7671{
7672 NOREF(pMixedCtx);
7673 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7674 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7675 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7676}
7677
7678
7679/**
7680 * Injects a general-protection (#GP) fault into the VM.
7681 *
7682 * @returns VBox status code (informational status code included).
7683 * @param pVCpu Pointer to the VMCPU.
7684 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7685 * out-of-sync. Make sure to update the required fields
7686 * before using them.
7687 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7688 * mode, i.e. in real-mode it's not valid).
7689 * @param u32ErrorCode The error code associated with the #GP.
7690 * @param fStepping Whether we're running in
7691 * hmR0VmxRunGuestCodeStep() and should return
7692 * VINF_EM_DBG_STEPPED if the event is injected
7693 * directly (register modified by us, not by
7694 * hardware on VM-entry).
7695 * @param puIntrState Pointer to the current guest interruptibility-state.
7696 * This interruptibility-state will be updated if
7697 * necessary. This cannot not be NULL.
7698 */
7699DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7700 bool fStepping, uint32_t *puIntrState)
7701{
7702 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7703 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7704 if (fErrorCodeValid)
7705 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7706 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7707 fStepping, puIntrState);
7708}
7709
7710
7711/**
7712 * Sets a general-protection (#GP) exception as pending-for-injection into the
7713 * VM.
7714 *
7715 * @param pVCpu Pointer to the VMCPU.
7716 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7717 * out-of-sync. Make sure to update the required fields
7718 * before using them.
7719 * @param u32ErrorCode The error code associated with the #GP.
7720 */
7721DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7722{
7723 NOREF(pMixedCtx);
7724 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7725 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7726 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7727 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7728}
7729
7730
7731/**
7732 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7733 *
7734 * @param pVCpu Pointer to the VMCPU.
7735 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7736 * out-of-sync. Make sure to update the required fields
7737 * before using them.
7738 * @param uVector The software interrupt vector number.
7739 * @param cbInstr The value of RIP that is to be pushed on the guest
7740 * stack.
7741 */
7742DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7743{
7744 NOREF(pMixedCtx);
7745 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7746 if ( uVector == X86_XCPT_BP
7747 || uVector == X86_XCPT_OF)
7748 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7749 else
7750 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7751 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7752}
7753
7754
7755/**
7756 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7757 * stack.
7758 *
7759 * @returns VBox status code (information status code included).
7760 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7761 * @param pVM Pointer to the VM.
7762 * @param pMixedCtx Pointer to the guest-CPU context.
7763 * @param uValue The value to push to the guest stack.
7764 */
7765DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7766{
7767 /*
7768 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7769 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7770 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7771 */
7772 if (pMixedCtx->sp == 1)
7773 return VINF_EM_RESET;
7774 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7775 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7776 AssertRCReturn(rc, rc);
7777 return rc;
7778}
7779
7780
7781/**
7782 * Injects an event into the guest upon VM-entry by updating the relevant fields
7783 * in the VM-entry area in the VMCS.
7784 *
7785 * @returns VBox status code (informational error codes included).
7786 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7787 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7788 *
7789 * @param pVCpu Pointer to the VMCPU.
7790 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7791 * be out-of-sync. Make sure to update the required
7792 * fields before using them.
7793 * @param u64IntInfo The VM-entry interruption-information field.
7794 * @param cbInstr The VM-entry instruction length in bytes (for
7795 * software interrupts, exceptions and privileged
7796 * software exceptions).
7797 * @param u32ErrCode The VM-entry exception error code.
7798 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7799 * @param puIntrState Pointer to the current guest interruptibility-state.
7800 * This interruptibility-state will be updated if
7801 * necessary. This cannot not be NULL.
7802 * @param fStepping Whether we're running in
7803 * hmR0VmxRunGuestCodeStep() and should return
7804 * VINF_EM_DBG_STEPPED if the event is injected
7805 * directly (register modified by us, not by
7806 * hardware on VM-entry).
7807 *
7808 * @remarks Requires CR0!
7809 * @remarks No-long-jump zone!!!
7810 */
7811static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7812 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *puIntrState)
7813{
7814 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7815 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7816 Assert(puIntrState);
7817 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7818
7819 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7820 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7821
7822#ifdef VBOX_STRICT
7823 /* Validate the error-code-valid bit for hardware exceptions. */
7824 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7825 {
7826 switch (uVector)
7827 {
7828 case X86_XCPT_PF:
7829 case X86_XCPT_DF:
7830 case X86_XCPT_TS:
7831 case X86_XCPT_NP:
7832 case X86_XCPT_SS:
7833 case X86_XCPT_GP:
7834 case X86_XCPT_AC:
7835 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7836 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7837 /* fallthru */
7838 default:
7839 break;
7840 }
7841 }
7842#endif
7843
7844 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7845 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7846 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7847
7848 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7849
7850 /* We require CR0 to check if the guest is in real-mode. */
7851 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7852 AssertRCReturn(rc, rc);
7853
7854 /*
7855 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7856 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7857 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7858 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7859 */
7860 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7861 {
7862 PVM pVM = pVCpu->CTX_SUFF(pVM);
7863 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7864 {
7865 Assert(PDMVmmDevHeapIsEnabled(pVM));
7866 Assert(pVM->hm.s.vmx.pRealModeTSS);
7867
7868 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7869 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7870 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7871 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7872 AssertRCReturn(rc, rc);
7873 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7874
7875 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7876 size_t const cbIdtEntry = sizeof(X86IDTR16);
7877 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7878 {
7879 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7880 if (uVector == X86_XCPT_DF)
7881 return VINF_EM_RESET;
7882
7883 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7884 if (uVector == X86_XCPT_GP)
7885 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
7886
7887 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7888 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7889 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
7890 fStepping, puIntrState);
7891 }
7892
7893 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7894 uint16_t uGuestIp = pMixedCtx->ip;
7895 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7896 {
7897 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7898 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7899 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7900 }
7901 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7902 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7903
7904 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7905 X86IDTR16 IdtEntry;
7906 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7907 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7908 AssertRCReturn(rc, rc);
7909
7910 /* Construct the stack frame for the interrupt/exception handler. */
7911 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7912 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7913 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7914 AssertRCReturn(rc, rc);
7915
7916 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7917 if (rc == VINF_SUCCESS)
7918 {
7919 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7920 pMixedCtx->rip = IdtEntry.offSel;
7921 pMixedCtx->cs.Sel = IdtEntry.uSel;
7922 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7923 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7924 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7925 && uVector == X86_XCPT_PF)
7926 pMixedCtx->cr2 = GCPtrFaultAddress;
7927
7928 /* If any other guest-state bits are changed here, make sure to update
7929 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7930 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7931 | HM_CHANGED_GUEST_RIP
7932 | HM_CHANGED_GUEST_RFLAGS
7933 | HM_CHANGED_GUEST_RSP);
7934
7935 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7936 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7937 {
7938 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7939 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7940 Log4(("Clearing inhibition due to STI.\n"));
7941 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7942 }
7943 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7944 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
7945
7946 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7947 it, if we are returning to ring-3 before executing guest code. */
7948 pVCpu->hm.s.Event.fPending = false;
7949
7950 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
7951 if (fStepping)
7952 rc = VINF_EM_DBG_STEPPED;
7953 }
7954 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
7955 return rc;
7956 }
7957
7958 /*
7959 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7960 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7961 */
7962 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7963 }
7964
7965 /* Validate. */
7966 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7967 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7968 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7969
7970 /* Inject. */
7971 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7972 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7973 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7974 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7975
7976 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7977 && uVector == X86_XCPT_PF)
7978 pMixedCtx->cr2 = GCPtrFaultAddress;
7979
7980 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7981 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7982
7983 AssertRCReturn(rc, rc);
7984 return rc;
7985}
7986
7987
7988/**
7989 * Clears the interrupt-window exiting control in the VMCS and if necessary
7990 * clears the current event in the VMCS as well.
7991 *
7992 * @returns VBox status code.
7993 * @param pVCpu Pointer to the VMCPU.
7994 *
7995 * @remarks Use this function only to clear events that have not yet been
7996 * delivered to the guest but are injected in the VMCS!
7997 * @remarks No-long-jump zone!!!
7998 */
7999static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
8000{
8001 int rc;
8002 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8003
8004 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8005 {
8006 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8007 Assert(!pVCpu->hm.s.Event.fPending);
8008 }
8009
8010 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8011 {
8012 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8013 Assert(!pVCpu->hm.s.Event.fPending);
8014 }
8015
8016 if (!pVCpu->hm.s.Event.fPending)
8017 return;
8018
8019#ifdef VBOX_STRICT
8020 uint32_t u32EntryInfo;
8021 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8022 AssertRC(rc);
8023 Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
8024#endif
8025
8026 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8027 AssertRC(rc);
8028
8029 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
8030 AssertRC(rc);
8031
8032 /* We deliberately don't clear "hm.s.Event.fPending" here, it's taken
8033 care of in hmR0VmxExitToRing3() converting the pending event to TRPM. */
8034}
8035
8036
8037/**
8038 * Enters the VT-x session.
8039 *
8040 * @returns VBox status code.
8041 * @param pVM Pointer to the VM.
8042 * @param pVCpu Pointer to the VMCPU.
8043 * @param pCpu Pointer to the CPU info struct.
8044 */
8045VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8046{
8047 AssertPtr(pVM);
8048 AssertPtr(pVCpu);
8049 Assert(pVM->hm.s.vmx.fSupported);
8050 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8051 NOREF(pCpu); NOREF(pVM);
8052
8053 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8054 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8055
8056#ifdef VBOX_STRICT
8057 /* Make sure we're in VMX root mode. */
8058 RTCCUINTREG u32HostCR4 = ASMGetCR4();
8059 if (!(u32HostCR4 & X86_CR4_VMXE))
8060 {
8061 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8062 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8063 }
8064#endif
8065
8066 /*
8067 * Load the VCPU's VMCS as the current (and active) one.
8068 */
8069 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8070 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8071 if (RT_FAILURE(rc))
8072 return rc;
8073
8074 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8075 pVCpu->hm.s.fLeaveDone = false;
8076 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8077
8078 return VINF_SUCCESS;
8079}
8080
8081
8082/**
8083 * The thread-context callback (only on platforms which support it).
8084 *
8085 * @param enmEvent The thread-context event.
8086 * @param pVCpu Pointer to the VMCPU.
8087 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8088 * @thread EMT(pVCpu)
8089 */
8090VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8091{
8092 NOREF(fGlobalInit);
8093
8094 switch (enmEvent)
8095 {
8096 case RTTHREADCTXEVENT_PREEMPTING:
8097 {
8098 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8099 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
8100 VMCPU_ASSERT_EMT(pVCpu);
8101
8102 PVM pVM = pVCpu->CTX_SUFF(pVM);
8103 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8104
8105 /* No longjmps (logger flushes, locks) in this fragile context. */
8106 VMMRZCallRing3Disable(pVCpu);
8107 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8108
8109 /*
8110 * Restore host-state (FPU, debug etc.)
8111 */
8112 if (!pVCpu->hm.s.fLeaveDone)
8113 {
8114 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8115 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8116 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
8117 pVCpu->hm.s.fLeaveDone = true;
8118 }
8119
8120 /* Leave HM context, takes care of local init (term). */
8121 int rc = HMR0LeaveCpu(pVCpu);
8122 AssertRC(rc); NOREF(rc);
8123
8124 /* Restore longjmp state. */
8125 VMMRZCallRing3Enable(pVCpu);
8126 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
8127 break;
8128 }
8129
8130 case RTTHREADCTXEVENT_RESUMED:
8131 {
8132 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8133 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
8134 VMCPU_ASSERT_EMT(pVCpu);
8135
8136 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8137 VMMRZCallRing3Disable(pVCpu);
8138 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8139
8140 /* Initialize the bare minimum state required for HM. This takes care of
8141 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8142 int rc = HMR0EnterCpu(pVCpu);
8143 AssertRC(rc);
8144 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8145
8146 /* Load the active VMCS as the current one. */
8147 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8148 {
8149 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8150 AssertRC(rc); NOREF(rc);
8151 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8152 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8153 }
8154 pVCpu->hm.s.fLeaveDone = false;
8155
8156 /* Restore longjmp state. */
8157 VMMRZCallRing3Enable(pVCpu);
8158 break;
8159 }
8160
8161 default:
8162 break;
8163 }
8164}
8165
8166
8167/**
8168 * Saves the host state in the VMCS host-state.
8169 * Sets up the VM-exit MSR-load area.
8170 *
8171 * The CPU state will be loaded from these fields on every successful VM-exit.
8172 *
8173 * @returns VBox status code.
8174 * @param pVM Pointer to the VM.
8175 * @param pVCpu Pointer to the VMCPU.
8176 *
8177 * @remarks No-long-jump zone!!!
8178 */
8179static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8180{
8181 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8182
8183 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8184 return VINF_SUCCESS;
8185
8186 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8187 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8188
8189 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8190 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8191
8192 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8193 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8194
8195 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8196 return rc;
8197}
8198
8199
8200/**
8201 * Saves the host state in the VMCS host-state.
8202 *
8203 * @returns VBox status code.
8204 * @param pVM Pointer to the VM.
8205 * @param pVCpu Pointer to the VMCPU.
8206 *
8207 * @remarks No-long-jump zone!!!
8208 */
8209VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8210{
8211 AssertPtr(pVM);
8212 AssertPtr(pVCpu);
8213
8214 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8215
8216 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8217 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8218 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8219 return hmR0VmxSaveHostState(pVM, pVCpu);
8220}
8221
8222
8223/**
8224 * Loads the guest state into the VMCS guest-state area.
8225 *
8226 * The will typically be done before VM-entry when the guest-CPU state and the
8227 * VMCS state may potentially be out of sync.
8228 *
8229 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8230 * VM-entry controls.
8231 * Sets up the appropriate VMX non-root function to execute guest code based on
8232 * the guest CPU mode.
8233 *
8234 * @returns VBox status code.
8235 * @param pVM Pointer to the VM.
8236 * @param pVCpu Pointer to the VMCPU.
8237 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8238 * out-of-sync. Make sure to update the required fields
8239 * before using them.
8240 *
8241 * @remarks No-long-jump zone!!!
8242 */
8243static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8244{
8245 AssertPtr(pVM);
8246 AssertPtr(pVCpu);
8247 AssertPtr(pMixedCtx);
8248 HMVMX_ASSERT_PREEMPT_SAFE();
8249
8250 VMMRZCallRing3Disable(pVCpu);
8251 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8252
8253 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8254
8255 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8256
8257 /* Determine real-on-v86 mode. */
8258 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8259 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8260 && CPUMIsGuestInRealModeEx(pMixedCtx))
8261 {
8262 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8263 }
8264
8265 /*
8266 * Load the guest-state into the VMCS.
8267 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8268 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8269 */
8270 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8271 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8272
8273 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8274 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8275 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8276
8277 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8278 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8279 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8280
8281 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8282 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8283
8284 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8285 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8286
8287 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8288 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8289 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8290
8291 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8292 determine we don't have to swap EFER after all. */
8293 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8294 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8295
8296 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8297 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8298
8299 /*
8300 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8301 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8302 */
8303 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8304 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8305
8306 /* Clear any unused and reserved bits. */
8307 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8308
8309 VMMRZCallRing3Enable(pVCpu);
8310
8311 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8312 return rc;
8313}
8314
8315
8316/**
8317 * Loads the state shared between the host and guest into the VMCS.
8318 *
8319 * @param pVM Pointer to the VM.
8320 * @param pVCpu Pointer to the VMCPU.
8321 * @param pCtx Pointer to the guest-CPU context.
8322 *
8323 * @remarks No-long-jump zone!!!
8324 */
8325static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8326{
8327 NOREF(pVM);
8328
8329 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8330 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8331
8332 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8333 {
8334 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8335 AssertRC(rc);
8336 }
8337
8338 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8339 {
8340 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8341 AssertRC(rc);
8342
8343 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8344 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8345 {
8346 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8347 AssertRC(rc);
8348 }
8349 }
8350
8351 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8352 {
8353#if HC_ARCH_BITS == 64
8354 if (pVM->hm.s.fAllow64BitGuests)
8355 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8356#endif
8357 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8358 }
8359
8360 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8361 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8362}
8363
8364
8365/**
8366 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8367 *
8368 * @param pVM Pointer to the VM.
8369 * @param pVCpu Pointer to the VMCPU.
8370 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8371 * out-of-sync. Make sure to update the required fields
8372 * before using them.
8373 */
8374DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8375{
8376 HMVMX_ASSERT_PREEMPT_SAFE();
8377
8378 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8379#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8380 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8381#endif
8382
8383 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8384 {
8385 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8386 AssertRC(rc);
8387 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8388 }
8389 else if (HMCPU_CF_VALUE(pVCpu))
8390 {
8391 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8392 AssertRC(rc);
8393 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8394 }
8395
8396 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8397 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8398 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8399 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8400}
8401
8402
8403/**
8404 * Does the preparations before executing guest code in VT-x.
8405 *
8406 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8407 * recompiler/IEM. We must be cautious what we do here regarding committing
8408 * guest-state information into the VMCS assuming we assuredly execute the
8409 * guest in VT-x mode.
8410 *
8411 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8412 * the common-state (TRPM/forceflags), we must undo those changes so that the
8413 * recompiler/IEM can (and should) use them when it resumes guest execution.
8414 * Otherwise such operations must be done when we can no longer exit to ring-3.
8415 *
8416 * @returns Strict VBox status code.
8417 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8418 * have been disabled.
8419 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8420 * double-fault into the guest.
8421 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8422 * dispatched directly.
8423 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8424 *
8425 * @param pVM Pointer to the VM.
8426 * @param pVCpu Pointer to the VMCPU.
8427 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8428 * out-of-sync. Make sure to update the required fields
8429 * before using them.
8430 * @param pVmxTransient Pointer to the VMX transient structure.
8431 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8432 * us ignore some of the reasons for returning to
8433 * ring-3, and return VINF_EM_DBG_STEPPED if event
8434 * dispatching took place.
8435 */
8436static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8437{
8438 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8439
8440#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8441 PGMRZDynMapFlushAutoSet(pVCpu);
8442#endif
8443
8444 /* Check force flag actions that might require us to go back to ring-3. */
8445 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
8446 if (rc != VINF_SUCCESS)
8447 return rc;
8448
8449#ifndef IEM_VERIFICATION_MODE_FULL
8450 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8451 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8452 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8453 {
8454 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8455 RTGCPHYS GCPhysApicBase;
8456 GCPhysApicBase = pMixedCtx->msrApicBase;
8457 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8458
8459 /* Unalias any existing mapping. */
8460 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8461 AssertRCReturn(rc, rc);
8462
8463 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8464 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
8465 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8466 AssertRCReturn(rc, rc);
8467
8468 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8469 }
8470#endif /* !IEM_VERIFICATION_MODE_FULL */
8471
8472 if (TRPMHasTrap(pVCpu))
8473 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8474 else if (!pVCpu->hm.s.Event.fPending)
8475 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8476
8477 /*
8478 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8479 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8480 */
8481 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, fStepping);
8482 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8483 {
8484 Assert(rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
8485 return rc;
8486 }
8487
8488 /*
8489 * Load the guest state bits, we can handle longjmps/getting preempted here.
8490 *
8491 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8492 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8493 * Hence, this needs to be done -after- injection of events.
8494 */
8495 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8496
8497 /*
8498 * No longjmps to ring-3 from this point on!!!
8499 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8500 * This also disables flushing of the R0-logger instance (if any).
8501 */
8502 VMMRZCallRing3Disable(pVCpu);
8503
8504 /*
8505 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8506 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8507 *
8508 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8509 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8510 *
8511 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8512 * executing guest code.
8513 */
8514 pVmxTransient->uEflags = ASMIntDisableFlags();
8515 if ( ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8516 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8517 && ( !fStepping /* Optimized for the non-stepping case, of course. */
8518 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8519 {
8520 hmR0VmxClearEventVmcs(pVCpu);
8521 ASMSetFlags(pVmxTransient->uEflags);
8522 VMMRZCallRing3Enable(pVCpu);
8523 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8524 return VINF_EM_RAW_TO_R3;
8525 }
8526
8527 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8528 {
8529 hmR0VmxClearEventVmcs(pVCpu);
8530 ASMSetFlags(pVmxTransient->uEflags);
8531 VMMRZCallRing3Enable(pVCpu);
8532 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8533 return VINF_EM_RAW_INTERRUPT;
8534 }
8535
8536 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8537 pVCpu->hm.s.Event.fPending = false;
8538
8539 return VINF_SUCCESS;
8540}
8541
8542
8543/**
8544 * Prepares to run guest code in VT-x and we've committed to doing so. This
8545 * means there is no backing out to ring-3 or anywhere else at this
8546 * point.
8547 *
8548 * @param pVM Pointer to the VM.
8549 * @param pVCpu Pointer to the VMCPU.
8550 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8551 * out-of-sync. Make sure to update the required fields
8552 * before using them.
8553 * @param pVmxTransient Pointer to the VMX transient structure.
8554 *
8555 * @remarks Called with preemption disabled.
8556 * @remarks No-long-jump zone!!!
8557 */
8558static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8559{
8560 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8561 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8562 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8563
8564 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8565 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8566
8567#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8568 if (!CPUMIsGuestFPUStateActive(pVCpu))
8569 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8570 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8571#endif
8572
8573 if ( pVCpu->hm.s.fUseGuestFpu
8574 && !CPUMIsGuestFPUStateActive(pVCpu))
8575 {
8576 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8577 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8578 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8579 }
8580
8581 /*
8582 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8583 */
8584 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8585 && pVCpu->hm.s.vmx.cMsrs > 0)
8586 {
8587 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8588 }
8589
8590 /*
8591 * Load the host state bits as we may've been preempted (only happens when
8592 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8593 */
8594 /** @todo Why should hmR0VmxSetupVMRunHandler() changing pfnStartVM have
8595 * any effect to the host state needing to be saved? */
8596 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8597 {
8598 /* This ASSUMES that pfnStartVM has been set up already. */
8599 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8600 AssertRC(rc);
8601 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8602 }
8603 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8604
8605 /*
8606 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8607 */
8608 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8609 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8610 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8611
8612 /* Store status of the shared guest-host state at the time of VM-entry. */
8613#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8614 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8615 {
8616 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8617 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8618 }
8619 else
8620#endif
8621 {
8622 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8623 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8624 }
8625 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8626
8627 /*
8628 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8629 */
8630 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8631 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8632
8633 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8634 RTCPUID idCurrentCpu = pCpu->idCpu;
8635 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8636 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8637 {
8638 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
8639 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8640 }
8641
8642 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8643 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8644 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8645 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8646
8647 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8648
8649 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8650 to start executing. */
8651
8652 /*
8653 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8654 */
8655 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8656 {
8657 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8658 {
8659 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8660 AssertRC(rc2);
8661 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8662 bool fMsrUpdated = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu),
8663 true /* fUpdateHostMsr */);
8664 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8665 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8666 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8667 }
8668 else
8669 {
8670 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8671 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8672 }
8673 }
8674
8675#ifdef VBOX_STRICT
8676 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8677 hmR0VmxCheckHostEferMsr(pVCpu);
8678 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8679#endif
8680#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8681 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8682 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8683 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8684#endif
8685}
8686
8687
8688/**
8689 * Performs some essential restoration of state after running guest code in
8690 * VT-x.
8691 *
8692 * @param pVM Pointer to the VM.
8693 * @param pVCpu Pointer to the VMCPU.
8694 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8695 * out-of-sync. Make sure to update the required fields
8696 * before using them.
8697 * @param pVmxTransient Pointer to the VMX transient structure.
8698 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8699 *
8700 * @remarks Called with interrupts disabled, and returns with interrups enabled!
8701 *
8702 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8703 * unconditionally when it is safe to do so.
8704 */
8705static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8706{
8707 NOREF(pVM);
8708
8709 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8710
8711 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8712 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8713 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8714 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8715 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8716 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8717
8718 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8719 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
8720
8721 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8722 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8723 Assert(!(ASMGetFlags() & X86_EFL_IF));
8724 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8725
8726#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8727 if (CPUMIsGuestFPUStateActive(pVCpu))
8728 {
8729 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8730 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8731 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8732 }
8733#endif
8734
8735#if HC_ARCH_BITS == 64
8736 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8737#endif
8738 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8739#ifdef VBOX_STRICT
8740 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8741#endif
8742 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8743 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8744
8745 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8746 uint32_t uExitReason;
8747 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8748 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8749 AssertRC(rc);
8750 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8751 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8752
8753 /* Update the VM-exit history array. */
8754 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8755
8756 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8757 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8758 {
8759 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8760 pVmxTransient->fVMEntryFailed));
8761 return;
8762 }
8763
8764 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8765 {
8766 /** @todo We can optimize this by only syncing with our force-flags when
8767 * really needed and keeping the VMCS state as it is for most
8768 * VM-exits. */
8769 /* Update the guest interruptibility-state from the VMCS. */
8770 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8771
8772#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8773 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8774 AssertRC(rc);
8775#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8776 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8777 AssertRC(rc);
8778#endif
8779
8780 /*
8781 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8782 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8783 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8784 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8785 */
8786 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8787 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8788 {
8789 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8790 AssertRC(rc);
8791 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8792 }
8793 }
8794}
8795
8796
8797/**
8798 * Runs the guest code using VT-x the normal way.
8799 *
8800 * @returns VBox status code.
8801 * @param pVM Pointer to the VM.
8802 * @param pVCpu Pointer to the VMCPU.
8803 * @param pCtx Pointer to the guest-CPU context.
8804 *
8805 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8806 */
8807static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8808{
8809 VMXTRANSIENT VmxTransient;
8810 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8811 int rc = VERR_INTERNAL_ERROR_5;
8812 uint32_t cLoops = 0;
8813
8814 for (;; cLoops++)
8815 {
8816 Assert(!HMR0SuspendPending());
8817 HMVMX_ASSERT_CPU_SAFE();
8818
8819 /* Preparatory work for running guest code, this may force us to return
8820 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8821 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8822 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8823 if (rc != VINF_SUCCESS)
8824 break;
8825
8826 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8827 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8828 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8829
8830 /* Restore any residual host-state and save any bits shared between host
8831 and guest into the guest-CPU state. Re-enables interrupts! */
8832 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8833
8834 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8835 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8836 {
8837 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8838 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8839 return rc;
8840 }
8841
8842 /* Profile the VM-exit. */
8843 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8844 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8845 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8846 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8847 HMVMX_START_EXIT_DISPATCH_PROF();
8848
8849 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8850 if (RT_UNLIKELY(VBOXVMM_R0_HMVMX_VMEXIT_ENABLED()))
8851 {
8852 hmR0VmxReadExitQualificationVmcs(pVCpu, &VmxTransient);
8853 hmR0VmxSaveGuestState(pVCpu, pCtx);
8854 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, VmxTransient.uExitReason, VmxTransient.uExitQualification);
8855 }
8856
8857 /* Handle the VM-exit. */
8858#ifdef HMVMX_USE_FUNCTION_TABLE
8859 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8860#else
8861 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8862#endif
8863 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8864 if (rc != VINF_SUCCESS)
8865 break;
8866 if (cLoops > pVM->hm.s.cMaxResumeLoops)
8867 {
8868 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8869 rc = VINF_EM_RAW_INTERRUPT;
8870 break;
8871 }
8872 }
8873
8874 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8875 return rc;
8876}
8877
8878
8879/**
8880 * Single steps guest code using VT-x.
8881 *
8882 * @returns VBox status code.
8883 * @param pVM Pointer to the VM.
8884 * @param pVCpu Pointer to the VMCPU.
8885 * @param pCtx Pointer to the guest-CPU context.
8886 *
8887 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8888 */
8889static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8890{
8891 VMXTRANSIENT VmxTransient;
8892 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8893 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8894 uint32_t cLoops = 0;
8895 uint16_t uCsStart = pCtx->cs.Sel;
8896 uint64_t uRipStart = pCtx->rip;
8897
8898 for (;; cLoops++)
8899 {
8900 Assert(!HMR0SuspendPending());
8901 HMVMX_ASSERT_CPU_SAFE();
8902
8903 /* Preparatory work for running guest code, this may force us to return
8904 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8905 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8906 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, true /* fStepping */);
8907 if (rcStrict != VINF_SUCCESS)
8908 break;
8909
8910 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8911 rcStrict = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8912 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8913
8914 /* Restore any residual host-state and save any bits shared between host
8915 and guest into the guest-CPU state. Re-enables interrupts! */
8916 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
8917
8918 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8919 if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
8920 {
8921 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8922 hmR0VmxReportWorldSwitchError(pVM, pVCpu, VBOXSTRICTRC_TODO(rcStrict), pCtx, &VmxTransient);
8923 return VBOXSTRICTRC_TODO(rcStrict);
8924 }
8925
8926 /* Profile the VM-exit. */
8927 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8928 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8929 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8930 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8931 HMVMX_START_EXIT_DISPATCH_PROF();
8932
8933 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8934 if (RT_UNLIKELY(VBOXVMM_R0_HMVMX_VMEXIT_ENABLED()))
8935 {
8936 hmR0VmxReadExitQualificationVmcs(pVCpu, &VmxTransient);
8937 hmR0VmxSaveGuestState(pVCpu, pCtx);
8938 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, VmxTransient.uExitReason, VmxTransient.uExitQualification);
8939 }
8940
8941 /* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitStep(). */
8942 rcStrict = hmR0VmxHandleExitStep(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, uCsStart, uRipStart);
8943 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8944 if (rcStrict != VINF_SUCCESS)
8945 break;
8946 if (cLoops > pVM->hm.s.cMaxResumeLoops)
8947 {
8948 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8949 rcStrict = VINF_EM_RAW_INTERRUPT;
8950 break;
8951 }
8952
8953 /*
8954 * Did the RIP change, if so, consider it a single step.
8955 * Otherwise, make sure one of the TFs gets set.
8956 */
8957 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8958 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8959 AssertRCReturn(rc2, rc2);
8960 if ( pCtx->rip != uRipStart
8961 || pCtx->cs.Sel != uCsStart)
8962 {
8963 rcStrict = VINF_EM_DBG_STEPPED;
8964 break;
8965 }
8966 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8967 }
8968
8969 /*
8970 * Clear the X86_EFL_TF if necessary.
8971 */
8972 if (pVCpu->hm.s.fClearTrapFlag)
8973 {
8974 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8975 AssertRCReturn(rc2, rc2);
8976 pVCpu->hm.s.fClearTrapFlag = false;
8977 pCtx->eflags.Bits.u1TF = 0;
8978 }
8979 /** @todo there seems to be issues with the resume flag when the monitor trap
8980 * flag is pending without being used. Seen early in bios init when
8981 * accessing APIC page in protected mode. */
8982
8983 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8984 return VBOXSTRICTRC_TODO(rcStrict);
8985}
8986
8987
8988/**
8989 * Runs the guest code using VT-x.
8990 *
8991 * @returns VBox status code.
8992 * @param pVM Pointer to the VM.
8993 * @param pVCpu Pointer to the VMCPU.
8994 * @param pCtx Pointer to the guest-CPU context.
8995 */
8996VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8997{
8998 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8999 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
9000 HMVMX_ASSERT_PREEMPT_SAFE();
9001
9002 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
9003
9004 int rc;
9005 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
9006 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
9007 else
9008 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
9009
9010 if (rc == VERR_EM_INTERPRETER)
9011 rc = VINF_EM_RAW_EMULATE_INSTR;
9012 else if (rc == VINF_EM_RESET)
9013 rc = VINF_EM_TRIPLE_FAULT;
9014
9015 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
9016 if (RT_FAILURE(rc2))
9017 {
9018 pVCpu->hm.s.u32HMError = rc;
9019 rc = rc2;
9020 }
9021 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
9022 return rc;
9023}
9024
9025
9026#ifndef HMVMX_USE_FUNCTION_TABLE
9027DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
9028{
9029#ifdef DEBUG_ramshankar
9030# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
9031# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
9032#endif
9033 int rc;
9034 switch (rcReason)
9035 {
9036 case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); */ rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9037 case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); */ rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9038 case VMX_EXIT_IO_INSTR: /* SVVMCS(); */ rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9039 case VMX_EXIT_CPUID: /* SVVMCS(); */ rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9040 case VMX_EXIT_RDTSC: /* SVVMCS(); */ rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9041 case VMX_EXIT_RDTSCP: /* SVVMCS(); */ rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9042 case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9043 case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); */ rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9044 case VMX_EXIT_MOV_CRX: /* SVVMCS(); */ rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9045 case VMX_EXIT_EXT_INT: /* SVVMCS(); */ rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9046 case VMX_EXIT_INT_WINDOW: /* SVVMCS(); */ rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9047 case VMX_EXIT_MWAIT: /* SVVMCS(); */ rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9048 case VMX_EXIT_MONITOR: /* SVVMCS(); */ rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9049 case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); */ rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9050 case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); */ rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9051 case VMX_EXIT_RDMSR: /* SVVMCS(); */ rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9052 case VMX_EXIT_WRMSR: /* SVVMCS(); */ rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9053 case VMX_EXIT_MOV_DRX: /* SVVMCS(); */ rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9054 case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); */ rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9055 case VMX_EXIT_HLT: /* SVVMCS(); */ rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9056 case VMX_EXIT_INVD: /* SVVMCS(); */ rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9057 case VMX_EXIT_INVLPG: /* SVVMCS(); */ rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9058 case VMX_EXIT_RSM: /* SVVMCS(); */ rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9059 case VMX_EXIT_MTF: /* SVVMCS(); */ rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9060 case VMX_EXIT_PAUSE: /* SVVMCS(); */ rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9061 case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9062 case VMX_EXIT_TR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9063 case VMX_EXIT_WBINVD: /* SVVMCS(); */ rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9064 case VMX_EXIT_XSETBV: /* SVVMCS(); */ rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9065 case VMX_EXIT_RDRAND: /* SVVMCS(); */ rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9066 case VMX_EXIT_INVPCID: /* SVVMCS(); */ rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9067 case VMX_EXIT_GETSEC: /* SVVMCS(); */ rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9068 case VMX_EXIT_RDPMC: /* SVVMCS(); */ rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9069 case VMX_EXIT_VMCALL: /* SVVMCS(); */ rc = hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9070
9071 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
9072 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
9073 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
9074 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
9075 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
9076 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
9077 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
9078 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
9079 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
9080
9081 case VMX_EXIT_VMCLEAR:
9082 case VMX_EXIT_VMLAUNCH:
9083 case VMX_EXIT_VMPTRLD:
9084 case VMX_EXIT_VMPTRST:
9085 case VMX_EXIT_VMREAD:
9086 case VMX_EXIT_VMRESUME:
9087 case VMX_EXIT_VMWRITE:
9088 case VMX_EXIT_VMXOFF:
9089 case VMX_EXIT_VMXON:
9090 case VMX_EXIT_INVEPT:
9091 case VMX_EXIT_INVVPID:
9092 case VMX_EXIT_VMFUNC:
9093 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
9094 break;
9095 default:
9096 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
9097 break;
9098 }
9099 return rc;
9100}
9101#endif /* !HMVMX_USE_FUNCTION_TABLE */
9102
9103
9104/**
9105 * Single-stepping VM-exit filtering.
9106 *
9107 * This is preprocessing the exits and deciding whether we've gotten far enough
9108 * to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit handling is
9109 * performed.
9110 *
9111 * @returns Strict VBox status code.
9112 * @param pVCpu The virtual CPU of the calling EMT.
9113 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9114 * out-of-sync. Make sure to update the required
9115 * fields before using them.
9116 * @param pVmxTransient Pointer to the VMX-transient structure.
9117 * @param uExitReason The VM-exit reason.
9118 */
9119DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitStep(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9120 uint32_t uExitReason, uint16_t uCsStart, uint64_t uRipStart)
9121{
9122 switch (uExitReason)
9123 {
9124 case VMX_EXIT_XCPT_OR_NMI:
9125 {
9126 /* Check for host NMI. */
9127 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9128 AssertRCReturn(rc2, rc2);
9129 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9130 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9131 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9132 /* fall thru */
9133 }
9134
9135 case VMX_EXIT_EPT_MISCONFIG:
9136 case VMX_EXIT_TRIPLE_FAULT:
9137 case VMX_EXIT_APIC_ACCESS:
9138 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9139 case VMX_EXIT_TASK_SWITCH:
9140
9141 /* Instruction specific VM-exits: */
9142 case VMX_EXIT_IO_INSTR:
9143 case VMX_EXIT_CPUID:
9144 case VMX_EXIT_RDTSC:
9145 case VMX_EXIT_RDTSCP:
9146 case VMX_EXIT_MOV_CRX:
9147 case VMX_EXIT_MWAIT:
9148 case VMX_EXIT_MONITOR:
9149 case VMX_EXIT_RDMSR:
9150 case VMX_EXIT_WRMSR:
9151 case VMX_EXIT_MOV_DRX:
9152 case VMX_EXIT_HLT:
9153 case VMX_EXIT_INVD:
9154 case VMX_EXIT_INVLPG:
9155 case VMX_EXIT_RSM:
9156 case VMX_EXIT_PAUSE:
9157 case VMX_EXIT_XDTR_ACCESS:
9158 case VMX_EXIT_TR_ACCESS:
9159 case VMX_EXIT_WBINVD:
9160 case VMX_EXIT_XSETBV:
9161 case VMX_EXIT_RDRAND:
9162 case VMX_EXIT_INVPCID:
9163 case VMX_EXIT_GETSEC:
9164 case VMX_EXIT_RDPMC:
9165 case VMX_EXIT_VMCALL:
9166 case VMX_EXIT_VMCLEAR:
9167 case VMX_EXIT_VMLAUNCH:
9168 case VMX_EXIT_VMPTRLD:
9169 case VMX_EXIT_VMPTRST:
9170 case VMX_EXIT_VMREAD:
9171 case VMX_EXIT_VMRESUME:
9172 case VMX_EXIT_VMWRITE:
9173 case VMX_EXIT_VMXOFF:
9174 case VMX_EXIT_VMXON:
9175 case VMX_EXIT_INVEPT:
9176 case VMX_EXIT_INVVPID:
9177 case VMX_EXIT_VMFUNC:
9178 {
9179 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9180 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9181 AssertRCReturn(rc2, rc2);
9182 if ( pMixedCtx->rip != uRipStart
9183 || pMixedCtx->cs.Sel != uCsStart)
9184 return VINF_EM_DBG_STEPPED;
9185 break;
9186 }
9187 }
9188
9189 /*
9190 * Normal processing.
9191 */
9192#ifdef HMVMX_USE_FUNCTION_TABLE
9193 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9194#else
9195 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9196#endif
9197}
9198
9199
9200#ifdef DEBUG
9201/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
9202# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
9203 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
9204
9205# define HMVMX_ASSERT_PREEMPT_CPUID() \
9206 do \
9207 { \
9208 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
9209 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
9210 } while (0)
9211
9212# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9213 do { \
9214 AssertPtr(pVCpu); \
9215 AssertPtr(pMixedCtx); \
9216 AssertPtr(pVmxTransient); \
9217 Assert(pVmxTransient->fVMEntryFailed == false); \
9218 Assert(ASMIntAreEnabled()); \
9219 HMVMX_ASSERT_PREEMPT_SAFE(); \
9220 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
9221 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
9222 HMVMX_ASSERT_PREEMPT_SAFE(); \
9223 if (VMMR0IsLogFlushDisabled(pVCpu)) \
9224 HMVMX_ASSERT_PREEMPT_CPUID(); \
9225 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9226 } while (0)
9227
9228# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
9229 do { \
9230 Log4Func(("\n")); \
9231 } while (0)
9232#else /* Release builds */
9233# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9234 do { \
9235 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9236 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
9237 } while (0)
9238# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
9239#endif
9240
9241
9242/**
9243 * Advances the guest RIP after reading it from the VMCS.
9244 *
9245 * @returns VBox status code.
9246 * @param pVCpu Pointer to the VMCPU.
9247 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9248 * out-of-sync. Make sure to update the required fields
9249 * before using them.
9250 * @param pVmxTransient Pointer to the VMX transient structure.
9251 *
9252 * @remarks No-long-jump zone!!!
9253 */
9254DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9255{
9256 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9257 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9258 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9259 AssertRCReturn(rc, rc);
9260
9261 pMixedCtx->rip += pVmxTransient->cbInstr;
9262 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9263
9264 /*
9265 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
9266 * pending debug exception field as it takes care of priority of events.
9267 *
9268 * See Intel spec. 32.2.1 "Debug Exceptions".
9269 */
9270 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
9271
9272 return rc;
9273}
9274
9275
9276/**
9277 * Tries to determine what part of the guest-state VT-x has deemed as invalid
9278 * and update error record fields accordingly.
9279 *
9280 * @return VMX_IGS_* return codes.
9281 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9282 * wrong with the guest state.
9283 *
9284 * @param pVM Pointer to the VM.
9285 * @param pVCpu Pointer to the VMCPU.
9286 * @param pCtx Pointer to the guest-CPU state.
9287 *
9288 * @remarks This function assumes our cache of the VMCS controls
9289 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9290 */
9291static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9292{
9293#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9294#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9295 uError = (err); \
9296 break; \
9297 } else do { } while (0)
9298
9299 int rc;
9300 uint32_t uError = VMX_IGS_ERROR;
9301 uint32_t u32Val;
9302 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9303
9304 do
9305 {
9306 /*
9307 * CR0.
9308 */
9309 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9310 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9311 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
9312 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9313 if (fUnrestrictedGuest)
9314 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
9315
9316 uint32_t u32GuestCR0;
9317 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
9318 AssertRCBreak(rc);
9319 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
9320 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
9321 if ( !fUnrestrictedGuest
9322 && (u32GuestCR0 & X86_CR0_PG)
9323 && !(u32GuestCR0 & X86_CR0_PE))
9324 {
9325 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9326 }
9327
9328 /*
9329 * CR4.
9330 */
9331 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9332 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9333
9334 uint32_t u32GuestCR4;
9335 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
9336 AssertRCBreak(rc);
9337 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
9338 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
9339
9340 /*
9341 * IA32_DEBUGCTL MSR.
9342 */
9343 uint64_t u64Val;
9344 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9345 AssertRCBreak(rc);
9346 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9347 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9348 {
9349 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9350 }
9351 uint64_t u64DebugCtlMsr = u64Val;
9352
9353#ifdef VBOX_STRICT
9354 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9355 AssertRCBreak(rc);
9356 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
9357#endif
9358 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
9359
9360 /*
9361 * RIP and RFLAGS.
9362 */
9363 uint32_t u32Eflags;
9364#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9365 if (HMVMX_IS_64BIT_HOST_MODE())
9366 {
9367 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9368 AssertRCBreak(rc);
9369 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9370 if ( !fLongModeGuest
9371 || !pCtx->cs.Attr.n.u1Long)
9372 {
9373 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9374 }
9375 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9376 * must be identical if the "IA-32e mode guest" VM-entry
9377 * control is 1 and CS.L is 1. No check applies if the
9378 * CPU supports 64 linear-address bits. */
9379
9380 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9381 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9382 AssertRCBreak(rc);
9383 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9384 VMX_IGS_RFLAGS_RESERVED);
9385 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9386 u32Eflags = u64Val;
9387 }
9388 else
9389#endif
9390 {
9391 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9392 AssertRCBreak(rc);
9393 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9394 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9395 }
9396
9397 if ( fLongModeGuest
9398 || ( fUnrestrictedGuest
9399 && !(u32GuestCR0 & X86_CR0_PE)))
9400 {
9401 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9402 }
9403
9404 uint32_t u32EntryInfo;
9405 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9406 AssertRCBreak(rc);
9407 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9408 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9409 {
9410 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9411 }
9412
9413 /*
9414 * 64-bit checks.
9415 */
9416#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9417 if (HMVMX_IS_64BIT_HOST_MODE())
9418 {
9419 if ( fLongModeGuest
9420 && !fUnrestrictedGuest)
9421 {
9422 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9423 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9424 }
9425
9426 if ( !fLongModeGuest
9427 && (u32GuestCR4 & X86_CR4_PCIDE))
9428 {
9429 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9430 }
9431
9432 /** @todo CR3 field must be such that bits 63:52 and bits in the range
9433 * 51:32 beyond the processor's physical-address width are 0. */
9434
9435 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9436 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9437 {
9438 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9439 }
9440
9441 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9442 AssertRCBreak(rc);
9443 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9444
9445 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9446 AssertRCBreak(rc);
9447 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9448 }
9449#endif
9450
9451 /*
9452 * PERF_GLOBAL MSR.
9453 */
9454 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
9455 {
9456 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9457 AssertRCBreak(rc);
9458 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9459 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9460 }
9461
9462 /*
9463 * PAT MSR.
9464 */
9465 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
9466 {
9467 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9468 AssertRCBreak(rc);
9469 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9470 for (unsigned i = 0; i < 8; i++)
9471 {
9472 uint8_t u8Val = (u64Val & 0xff);
9473 if ( u8Val != 0 /* UC */
9474 && u8Val != 1 /* WC */
9475 && u8Val != 4 /* WT */
9476 && u8Val != 5 /* WP */
9477 && u8Val != 6 /* WB */
9478 && u8Val != 7 /* UC- */)
9479 {
9480 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9481 }
9482 u64Val >>= 8;
9483 }
9484 }
9485
9486 /*
9487 * EFER MSR.
9488 */
9489 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
9490 {
9491 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9492 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9493 AssertRCBreak(rc);
9494 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9495 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9496 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
9497 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9498 HMVMX_CHECK_BREAK( fUnrestrictedGuest
9499 || !(u32GuestCR0 & X86_CR0_PG)
9500 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9501 VMX_IGS_EFER_LMA_LME_MISMATCH);
9502 }
9503
9504 /*
9505 * Segment registers.
9506 */
9507 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9508 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9509 if (!(u32Eflags & X86_EFL_VM))
9510 {
9511 /* CS */
9512 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9513 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9514 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9515 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9516 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9517 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9518 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9519 /* CS cannot be loaded with NULL in protected mode. */
9520 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9521 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9522 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
9523 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9524 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
9525 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9526 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9527 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9528 else
9529 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9530
9531 /* SS */
9532 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9533 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9534 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9535 if ( !(pCtx->cr0 & X86_CR0_PE)
9536 || pCtx->cs.Attr.n.u4Type == 3)
9537 {
9538 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9539 }
9540 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9541 {
9542 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9543 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9544 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9545 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9546 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9547 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9548 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9549 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9550 }
9551
9552 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
9553 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9554 {
9555 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9556 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9557 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9558 || pCtx->ds.Attr.n.u4Type > 11
9559 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9560 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9561 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9562 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9563 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9564 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9565 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9566 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9567 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9568 }
9569 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9570 {
9571 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9572 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9573 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9574 || pCtx->es.Attr.n.u4Type > 11
9575 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9576 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9577 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9578 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9579 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9580 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9581 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9582 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9583 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9584 }
9585 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9586 {
9587 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9588 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9589 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9590 || pCtx->fs.Attr.n.u4Type > 11
9591 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9592 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9593 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9594 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9595 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9596 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9597 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9598 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9599 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9600 }
9601 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9602 {
9603 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9604 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9605 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9606 || pCtx->gs.Attr.n.u4Type > 11
9607 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9608 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9609 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9610 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9611 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9612 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9613 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9614 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9615 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9616 }
9617 /* 64-bit capable CPUs. */
9618#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9619 if (HMVMX_IS_64BIT_HOST_MODE())
9620 {
9621 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9622 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9623 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9624 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9625 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9626 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9627 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9628 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9629 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9630 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9631 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9632 }
9633#endif
9634 }
9635 else
9636 {
9637 /* V86 mode checks. */
9638 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9639 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9640 {
9641 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9642 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9643 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9644 }
9645 else
9646 {
9647 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9648 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9649 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9650 }
9651
9652 /* CS */
9653 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9654 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9655 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9656 /* SS */
9657 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9658 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9659 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9660 /* DS */
9661 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9662 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9663 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9664 /* ES */
9665 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9666 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9667 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9668 /* FS */
9669 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9670 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9671 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9672 /* GS */
9673 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9674 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9675 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9676 /* 64-bit capable CPUs. */
9677#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9678 if (HMVMX_IS_64BIT_HOST_MODE())
9679 {
9680 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9681 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9682 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9683 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9684 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9685 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9686 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9687 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9688 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9689 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9690 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9691 }
9692#endif
9693 }
9694
9695 /*
9696 * TR.
9697 */
9698 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9699 /* 64-bit capable CPUs. */
9700#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9701 if (HMVMX_IS_64BIT_HOST_MODE())
9702 {
9703 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9704 }
9705#endif
9706 if (fLongModeGuest)
9707 {
9708 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9709 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9710 }
9711 else
9712 {
9713 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9714 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9715 VMX_IGS_TR_ATTR_TYPE_INVALID);
9716 }
9717 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9718 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9719 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9720 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9721 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9722 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9723 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9724 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9725
9726 /*
9727 * GDTR and IDTR.
9728 */
9729#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9730 if (HMVMX_IS_64BIT_HOST_MODE())
9731 {
9732 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9733 AssertRCBreak(rc);
9734 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9735
9736 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9737 AssertRCBreak(rc);
9738 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9739 }
9740#endif
9741
9742 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9743 AssertRCBreak(rc);
9744 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9745
9746 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9747 AssertRCBreak(rc);
9748 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9749
9750 /*
9751 * Guest Non-Register State.
9752 */
9753 /* Activity State. */
9754 uint32_t u32ActivityState;
9755 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9756 AssertRCBreak(rc);
9757 HMVMX_CHECK_BREAK( !u32ActivityState
9758 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9759 VMX_IGS_ACTIVITY_STATE_INVALID);
9760 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9761 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9762 uint32_t u32IntrState;
9763 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9764 AssertRCBreak(rc);
9765 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9766 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9767 {
9768 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9769 }
9770
9771 /** @todo Activity state and injecting interrupts. Left as a todo since we
9772 * currently don't use activity states but ACTIVE. */
9773
9774 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9775 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9776
9777 /* Guest interruptibility-state. */
9778 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9779 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9780 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9781 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9782 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9783 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9784 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9785 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9786 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9787 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9788 {
9789 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9790 {
9791 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9792 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9793 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9794 }
9795 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9796 {
9797 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9798 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9799 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9800 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9801 }
9802 }
9803 /** @todo Assumes the processor is not in SMM. */
9804 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9805 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9806 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9807 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9808 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9809 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9810 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9811 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9812 {
9813 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9814 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9815 }
9816
9817 /* Pending debug exceptions. */
9818 if (HMVMX_IS_64BIT_HOST_MODE())
9819 {
9820 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9821 AssertRCBreak(rc);
9822 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9823 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9824 u32Val = u64Val; /* For pending debug exceptions checks below. */
9825 }
9826 else
9827 {
9828 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9829 AssertRCBreak(rc);
9830 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9831 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9832 }
9833
9834 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9835 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9836 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9837 {
9838 if ( (u32Eflags & X86_EFL_TF)
9839 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9840 {
9841 /* Bit 14 is PendingDebug.BS. */
9842 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9843 }
9844 if ( !(u32Eflags & X86_EFL_TF)
9845 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9846 {
9847 /* Bit 14 is PendingDebug.BS. */
9848 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9849 }
9850 }
9851
9852 /* VMCS link pointer. */
9853 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9854 AssertRCBreak(rc);
9855 if (u64Val != UINT64_C(0xffffffffffffffff))
9856 {
9857 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9858 /** @todo Bits beyond the processor's physical-address width MBZ. */
9859 /** @todo 32-bit located in memory referenced by value of this field (as a
9860 * physical address) must contain the processor's VMCS revision ID. */
9861 /** @todo SMM checks. */
9862 }
9863
9864 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9865 * not using Nested Paging? */
9866 if ( pVM->hm.s.fNestedPaging
9867 && !fLongModeGuest
9868 && CPUMIsGuestInPAEModeEx(pCtx))
9869 {
9870 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9871 AssertRCBreak(rc);
9872 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9873
9874 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9875 AssertRCBreak(rc);
9876 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9877
9878 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9879 AssertRCBreak(rc);
9880 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9881
9882 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9883 AssertRCBreak(rc);
9884 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9885 }
9886
9887 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9888 if (uError == VMX_IGS_ERROR)
9889 uError = VMX_IGS_REASON_NOT_FOUND;
9890 } while (0);
9891
9892 pVCpu->hm.s.u32HMError = uError;
9893 return uError;
9894
9895#undef HMVMX_ERROR_BREAK
9896#undef HMVMX_CHECK_BREAK
9897}
9898
9899/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9900/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9901/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9902
9903/** @name VM-exit handlers.
9904 * @{
9905 */
9906
9907/**
9908 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9909 */
9910HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9911{
9912 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9913 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9914 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9915 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9916 return VINF_SUCCESS;
9917 return VINF_EM_RAW_INTERRUPT;
9918}
9919
9920
9921/**
9922 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9923 */
9924HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9925{
9926 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9927 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9928
9929 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9930 AssertRCReturn(rc, rc);
9931
9932 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9933 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9934 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9935 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9936
9937 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9938 {
9939 /*
9940 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9941 * anything we inject is not going to cause a VM-exit directly for the event being injected.
9942 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9943 *
9944 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9945 */
9946 VMXDispatchHostNmi();
9947 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9948 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9949 return VINF_SUCCESS;
9950 }
9951
9952 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9953 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9954 if (RT_UNLIKELY(rc != VINF_SUCCESS))
9955 {
9956 if (rc == VINF_HM_DOUBLE_FAULT)
9957 rc = VINF_SUCCESS;
9958 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9959 return rc;
9960 }
9961
9962 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9963 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9964 switch (uIntType)
9965 {
9966 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9967 Assert(uVector == X86_XCPT_DB);
9968 /* no break */
9969 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9970 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9971 /* no break */
9972 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9973 {
9974 switch (uVector)
9975 {
9976 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9977 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9978 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9979 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9980 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9981 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9982#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9983 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9984 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9985 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9986 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9987 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9988 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9989 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9990 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9991 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9992 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9993 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
9994 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9995#endif
9996 default:
9997 {
9998 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9999 AssertRCReturn(rc, rc);
10000
10001 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
10002 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10003 {
10004 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
10005 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
10006 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
10007
10008 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10009 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10010 AssertRCReturn(rc, rc);
10011 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
10012 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
10013 0 /* GCPtrFaultAddress */);
10014 AssertRCReturn(rc, rc);
10015 }
10016 else
10017 {
10018 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
10019 pVCpu->hm.s.u32HMError = uVector;
10020 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10021 }
10022 break;
10023 }
10024 }
10025 break;
10026 }
10027
10028 default:
10029 {
10030 pVCpu->hm.s.u32HMError = uExitIntInfo;
10031 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
10032 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
10033 break;
10034 }
10035 }
10036 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10037 return rc;
10038}
10039
10040
10041/**
10042 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
10043 */
10044HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10045{
10046 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10047
10048 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
10049 hmR0VmxClearIntWindowExitVmcs(pVCpu);
10050
10051 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
10052 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
10053 return VINF_SUCCESS;
10054}
10055
10056
10057/**
10058 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
10059 */
10060HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10061{
10062 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10063 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
10064 {
10065 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
10066 HMVMX_RETURN_UNEXPECTED_EXIT();
10067 }
10068
10069 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
10070
10071 /*
10072 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
10073 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
10074 */
10075 uint32_t uIntrState = 0;
10076 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10077 AssertRCReturn(rc, rc);
10078
10079 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
10080 if ( fBlockSti
10081 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
10082 {
10083 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10084 }
10085
10086 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
10087 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
10088
10089 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
10090 return VINF_SUCCESS;
10091}
10092
10093
10094/**
10095 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
10096 */
10097HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10098{
10099 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10100 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
10101 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10102}
10103
10104
10105/**
10106 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
10107 */
10108HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10109{
10110 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10111 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
10112 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10113}
10114
10115
10116/**
10117 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
10118 */
10119HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10120{
10121 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10122 PVM pVM = pVCpu->CTX_SUFF(pVM);
10123 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10124 if (RT_LIKELY(rc == VINF_SUCCESS))
10125 {
10126 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10127 Assert(pVmxTransient->cbInstr == 2);
10128 }
10129 else
10130 {
10131 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
10132 rc = VERR_EM_INTERPRETER;
10133 }
10134 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
10135 return rc;
10136}
10137
10138
10139/**
10140 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
10141 */
10142HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10143{
10144 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10145 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
10146 AssertRCReturn(rc, rc);
10147
10148 if (pMixedCtx->cr4 & X86_CR4_SMXE)
10149 return VINF_EM_RAW_EMULATE_INSTR;
10150
10151 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
10152 HMVMX_RETURN_UNEXPECTED_EXIT();
10153}
10154
10155
10156/**
10157 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
10158 */
10159HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10160{
10161 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10162 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10163 AssertRCReturn(rc, rc);
10164
10165 PVM pVM = pVCpu->CTX_SUFF(pVM);
10166 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10167 if (RT_LIKELY(rc == VINF_SUCCESS))
10168 {
10169 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10170 Assert(pVmxTransient->cbInstr == 2);
10171 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10172 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10173 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10174 }
10175 else
10176 rc = VERR_EM_INTERPRETER;
10177 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10178 return rc;
10179}
10180
10181
10182/**
10183 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
10184 */
10185HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10186{
10187 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10188 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10189 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
10190 AssertRCReturn(rc, rc);
10191
10192 PVM pVM = pVCpu->CTX_SUFF(pVM);
10193 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
10194 if (RT_LIKELY(rc == VINF_SUCCESS))
10195 {
10196 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10197 Assert(pVmxTransient->cbInstr == 3);
10198 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10199 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10200 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10201 }
10202 else
10203 {
10204 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
10205 rc = VERR_EM_INTERPRETER;
10206 }
10207 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10208 return rc;
10209}
10210
10211
10212/**
10213 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
10214 */
10215HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10216{
10217 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10218 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10219 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
10220 AssertRCReturn(rc, rc);
10221
10222 PVM pVM = pVCpu->CTX_SUFF(pVM);
10223 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10224 if (RT_LIKELY(rc == VINF_SUCCESS))
10225 {
10226 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10227 Assert(pVmxTransient->cbInstr == 2);
10228 }
10229 else
10230 {
10231 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
10232 rc = VERR_EM_INTERPRETER;
10233 }
10234 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
10235 return rc;
10236}
10237
10238
10239/**
10240 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
10241 */
10242HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10243{
10244 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10245
10246 int rc = VERR_NOT_SUPPORTED;
10247 if (GIMAreHypercallsEnabled(pVCpu))
10248 {
10249 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10250 AssertRCReturn(rc, rc);
10251
10252 rc = GIMHypercall(pVCpu, pMixedCtx);
10253 }
10254 if (rc != VINF_SUCCESS)
10255 {
10256 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10257 rc = VINF_SUCCESS;
10258 }
10259
10260 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
10261 return rc;
10262}
10263
10264
10265/**
10266 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
10267 */
10268HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10269{
10270 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10271 PVM pVM = pVCpu->CTX_SUFF(pVM);
10272 Assert(!pVM->hm.s.fNestedPaging);
10273
10274 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10275 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10276 AssertRCReturn(rc, rc);
10277
10278 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
10279 rc = VBOXSTRICTRC_VAL(rc2);
10280 if (RT_LIKELY(rc == VINF_SUCCESS))
10281 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10282 else
10283 {
10284 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
10285 pVmxTransient->uExitQualification, rc));
10286 }
10287 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
10288 return rc;
10289}
10290
10291
10292/**
10293 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
10294 */
10295HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10296{
10297 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10298 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10299 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10300 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10301 AssertRCReturn(rc, rc);
10302
10303 PVM pVM = pVCpu->CTX_SUFF(pVM);
10304 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10305 if (RT_LIKELY(rc == VINF_SUCCESS))
10306 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10307 else
10308 {
10309 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
10310 rc = VERR_EM_INTERPRETER;
10311 }
10312 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
10313 return rc;
10314}
10315
10316
10317/**
10318 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
10319 */
10320HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10321{
10322 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10323 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10324 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10325 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10326 AssertRCReturn(rc, rc);
10327
10328 PVM pVM = pVCpu->CTX_SUFF(pVM);
10329 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10330 rc = VBOXSTRICTRC_VAL(rc2);
10331 if (RT_LIKELY( rc == VINF_SUCCESS
10332 || rc == VINF_EM_HALT))
10333 {
10334 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10335 AssertRCReturn(rc3, rc3);
10336
10337 if ( rc == VINF_EM_HALT
10338 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
10339 {
10340 rc = VINF_SUCCESS;
10341 }
10342 }
10343 else
10344 {
10345 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
10346 rc = VERR_EM_INTERPRETER;
10347 }
10348 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
10349 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
10350 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
10351 return rc;
10352}
10353
10354
10355/**
10356 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
10357 */
10358HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10359{
10360 /*
10361 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
10362 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
10363 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
10364 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
10365 */
10366 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10367 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10368 HMVMX_RETURN_UNEXPECTED_EXIT();
10369}
10370
10371
10372/**
10373 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
10374 */
10375HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10376{
10377 /*
10378 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
10379 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
10380 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
10381 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
10382 */
10383 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10384 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10385 HMVMX_RETURN_UNEXPECTED_EXIT();
10386}
10387
10388
10389/**
10390 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
10391 */
10392HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10393{
10394 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
10395 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10396 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10397 HMVMX_RETURN_UNEXPECTED_EXIT();
10398}
10399
10400
10401/**
10402 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
10403 */
10404HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10405{
10406 /*
10407 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
10408 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
10409 * See Intel spec. 25.3 "Other Causes of VM-exits".
10410 */
10411 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10412 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10413 HMVMX_RETURN_UNEXPECTED_EXIT();
10414}
10415
10416
10417/**
10418 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
10419 * VM-exit.
10420 */
10421HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10422{
10423 /*
10424 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
10425 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
10426 *
10427 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
10428 * See Intel spec. "23.8 Restrictions on VMX operation".
10429 */
10430 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10431 return VINF_SUCCESS;
10432}
10433
10434
10435/**
10436 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
10437 * VM-exit.
10438 */
10439HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10440{
10441 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10442 return VINF_EM_RESET;
10443}
10444
10445
10446/**
10447 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
10448 */
10449HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10450{
10451 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10452 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
10453 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10454 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10455 AssertRCReturn(rc, rc);
10456
10457 pMixedCtx->rip++;
10458 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10459 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
10460 rc = VINF_SUCCESS;
10461 else
10462 rc = VINF_EM_HALT;
10463
10464 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10465 if (rc != VINF_SUCCESS)
10466 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
10467 return rc;
10468}
10469
10470
10471/**
10472 * VM-exit handler for instructions that result in a #UD exception delivered to
10473 * the guest.
10474 */
10475HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10476{
10477 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10478 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10479 return VINF_SUCCESS;
10480}
10481
10482
10483/**
10484 * VM-exit handler for expiry of the VMX preemption timer.
10485 */
10486HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10487{
10488 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10489
10490 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
10491 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10492
10493 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
10494 PVM pVM = pVCpu->CTX_SUFF(pVM);
10495 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
10496 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
10497 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
10498}
10499
10500
10501/**
10502 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
10503 */
10504HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10505{
10506 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10507
10508 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
10509 /** @todo check if XSETBV is supported by the recompiler. */
10510 return VERR_EM_INTERPRETER;
10511}
10512
10513
10514/**
10515 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
10516 */
10517HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10518{
10519 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10520
10521 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
10522 /** @todo implement EMInterpretInvpcid() */
10523 return VERR_EM_INTERPRETER;
10524}
10525
10526
10527/**
10528 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
10529 * Error VM-exit.
10530 */
10531HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10532{
10533 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10534 AssertRCReturn(rc, rc);
10535
10536 rc = hmR0VmxCheckVmcsCtls(pVCpu);
10537 AssertRCReturn(rc, rc);
10538
10539 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10540 NOREF(uInvalidReason);
10541
10542#ifdef VBOX_STRICT
10543 uint32_t uIntrState;
10544 HMVMXHCUINTREG uHCReg;
10545 uint64_t u64Val;
10546 uint32_t u32Val;
10547
10548 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
10549 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
10550 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
10551 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10552 AssertRCReturn(rc, rc);
10553
10554 Log4(("uInvalidReason %u\n", uInvalidReason));
10555 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
10556 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
10557 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
10558 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
10559
10560 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
10561 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
10562 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
10563 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
10564 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
10565 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10566 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
10567 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
10568 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
10569 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10570 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
10571 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
10572#else
10573 NOREF(pVmxTransient);
10574#endif
10575
10576 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10577 return VERR_VMX_INVALID_GUEST_STATE;
10578}
10579
10580
10581/**
10582 * VM-exit handler for VM-entry failure due to an MSR-load
10583 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
10584 */
10585HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10586{
10587 NOREF(pVmxTransient);
10588 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10589 HMVMX_RETURN_UNEXPECTED_EXIT();
10590}
10591
10592
10593/**
10594 * VM-exit handler for VM-entry failure due to a machine-check event
10595 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
10596 */
10597HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10598{
10599 NOREF(pVmxTransient);
10600 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10601 HMVMX_RETURN_UNEXPECTED_EXIT();
10602}
10603
10604
10605/**
10606 * VM-exit handler for all undefined reasons. Should never ever happen.. in
10607 * theory.
10608 */
10609HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10610{
10611 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
10612 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
10613 return VERR_VMX_UNDEFINED_EXIT_CODE;
10614}
10615
10616
10617/**
10618 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
10619 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
10620 * Conditional VM-exit.
10621 */
10622HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10623{
10624 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10625
10626 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
10627 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
10628 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
10629 return VERR_EM_INTERPRETER;
10630 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10631 HMVMX_RETURN_UNEXPECTED_EXIT();
10632}
10633
10634
10635/**
10636 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
10637 */
10638HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10639{
10640 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10641
10642 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
10643 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
10644 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
10645 return VERR_EM_INTERPRETER;
10646 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10647 HMVMX_RETURN_UNEXPECTED_EXIT();
10648}
10649
10650
10651/**
10652 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
10653 */
10654HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10655{
10656 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10657
10658 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
10659 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10660 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10661 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10662 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10663 {
10664 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10665 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10666 }
10667 AssertRCReturn(rc, rc);
10668 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10669
10670#ifdef VBOX_STRICT
10671 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
10672 {
10673 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
10674 && pMixedCtx->ecx != MSR_K6_EFER)
10675 {
10676 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10677 HMVMX_RETURN_UNEXPECTED_EXIT();
10678 }
10679# if HC_ARCH_BITS == 64
10680 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
10681 && hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10682 {
10683 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10684 HMVMX_RETURN_UNEXPECTED_EXIT();
10685 }
10686# endif
10687 }
10688#endif
10689
10690 PVM pVM = pVCpu->CTX_SUFF(pVM);
10691 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10692 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
10693 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
10694 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
10695 if (RT_LIKELY(rc == VINF_SUCCESS))
10696 {
10697 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10698 Assert(pVmxTransient->cbInstr == 2);
10699 }
10700 return rc;
10701}
10702
10703
10704/**
10705 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10706 */
10707HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10708{
10709 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10710 PVM pVM = pVCpu->CTX_SUFF(pVM);
10711 int rc = VINF_SUCCESS;
10712
10713 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
10714 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10715 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10716 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10717 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10718 {
10719 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10720 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10721 }
10722 AssertRCReturn(rc, rc);
10723 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
10724
10725 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10726 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10727 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10728
10729 if (RT_LIKELY(rc == VINF_SUCCESS))
10730 {
10731 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10732
10733 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
10734 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10735 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10736 {
10737 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10738 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10739 EMInterpretWrmsr() changes it. */
10740 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10741 }
10742 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10743 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10744 else if (pMixedCtx->ecx == MSR_K6_EFER)
10745 {
10746 /*
10747 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
10748 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
10749 * the other bits as well, SCE and NXE. See @bugref{7368}.
10750 */
10751 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
10752 }
10753
10754 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10755 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10756 {
10757 switch (pMixedCtx->ecx)
10758 {
10759 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10760 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10761 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10762 case MSR_K8_FS_BASE: /* no break */
10763 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10764 case MSR_K6_EFER: /* already handled above */ break;
10765 default:
10766 {
10767 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10768 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10769#if HC_ARCH_BITS == 64
10770 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10771 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10772#endif
10773 break;
10774 }
10775 }
10776 }
10777#ifdef VBOX_STRICT
10778 else
10779 {
10780 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10781 switch (pMixedCtx->ecx)
10782 {
10783 case MSR_IA32_SYSENTER_CS:
10784 case MSR_IA32_SYSENTER_EIP:
10785 case MSR_IA32_SYSENTER_ESP:
10786 case MSR_K8_FS_BASE:
10787 case MSR_K8_GS_BASE:
10788 {
10789 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10790 HMVMX_RETURN_UNEXPECTED_EXIT();
10791 }
10792
10793 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10794 default:
10795 {
10796 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10797 {
10798 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
10799 if (pMixedCtx->ecx != MSR_K6_EFER)
10800 {
10801 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10802 pMixedCtx->ecx));
10803 HMVMX_RETURN_UNEXPECTED_EXIT();
10804 }
10805 }
10806
10807#if HC_ARCH_BITS == 64
10808 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10809 {
10810 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10811 HMVMX_RETURN_UNEXPECTED_EXIT();
10812 }
10813#endif
10814 break;
10815 }
10816 }
10817 }
10818#endif /* VBOX_STRICT */
10819 }
10820 return rc;
10821}
10822
10823
10824/**
10825 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10826 */
10827HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10828{
10829 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10830
10831 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10832 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10833 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10834 return VERR_EM_INTERPRETER;
10835 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10836 HMVMX_RETURN_UNEXPECTED_EXIT();
10837}
10838
10839
10840/**
10841 * VM-exit handler for when the TPR value is lowered below the specified
10842 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10843 */
10844HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10845{
10846 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10847 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10848
10849 /*
10850 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10851 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10852 * resume guest execution.
10853 */
10854 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10855 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10856 return VINF_SUCCESS;
10857}
10858
10859
10860/**
10861 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10862 * VM-exit.
10863 *
10864 * @retval VINF_SUCCESS when guest execution can continue.
10865 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10866 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10867 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10868 * recompiler.
10869 */
10870HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10871{
10872 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10873 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10874 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10875 AssertRCReturn(rc, rc);
10876
10877 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
10878 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10879 PVM pVM = pVCpu->CTX_SUFF(pVM);
10880 switch (uAccessType)
10881 {
10882 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10883 {
10884#if 0
10885 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10886 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10887#else
10888 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10889 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10890 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10891#endif
10892 AssertRCReturn(rc, rc);
10893
10894 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10895 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10896 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10897 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
10898
10899 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10900 {
10901 case 0: /* CR0 */
10902 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10903 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10904 break;
10905 case 2: /* CR2 */
10906 /* Nothing to do here, CR2 it's not part of the VMCS. */
10907 break;
10908 case 3: /* CR3 */
10909 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10910 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10911 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10912 break;
10913 case 4: /* CR4 */
10914 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10915 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10916 break;
10917 case 8: /* CR8 */
10918 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10919 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10920 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10921 break;
10922 default:
10923 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10924 break;
10925 }
10926
10927 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10928 break;
10929 }
10930
10931 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10932 {
10933 /* EMInterpretCRxRead() requires EFER MSR, CS. */
10934 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10935 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10936 AssertRCReturn(rc, rc);
10937 Assert( !pVM->hm.s.fNestedPaging
10938 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10939 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10940
10941 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10942 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10943 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10944
10945 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10946 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10947 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10948 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10949 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10950 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10951 break;
10952 }
10953
10954 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10955 {
10956 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10957 AssertRCReturn(rc, rc);
10958 rc = EMInterpretCLTS(pVM, pVCpu);
10959 AssertRCReturn(rc, rc);
10960 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10961 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10962 Log4(("CRX CLTS write rc=%d\n", rc));
10963 break;
10964 }
10965
10966 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10967 {
10968 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10969 AssertRCReturn(rc, rc);
10970 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10971 if (RT_LIKELY(rc == VINF_SUCCESS))
10972 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10973 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10974 Log4(("CRX LMSW write rc=%d\n", rc));
10975 break;
10976 }
10977
10978 default:
10979 {
10980 AssertMsgFailed(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType));
10981 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10982 }
10983 }
10984
10985 /* Validate possible error codes. */
10986 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
10987 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10988 if (RT_SUCCESS(rc))
10989 {
10990 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10991 AssertRCReturn(rc2, rc2);
10992 }
10993
10994 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10995 return rc;
10996}
10997
10998
10999/**
11000 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
11001 * VM-exit.
11002 */
11003HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11004{
11005 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11006 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
11007
11008 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11009 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11010 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11011 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
11012 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
11013 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
11014 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
11015 AssertRCReturn(rc2, rc2);
11016
11017 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
11018 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
11019 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
11020 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
11021 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
11022 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
11023 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11024 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
11025
11026 /* I/O operation lookup arrays. */
11027 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
11028 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
11029
11030 VBOXSTRICTRC rcStrict;
11031 uint32_t const cbValue = s_aIOSizes[uIOWidth];
11032 uint32_t const cbInstr = pVmxTransient->cbInstr;
11033 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
11034 PVM pVM = pVCpu->CTX_SUFF(pVM);
11035 if (fIOString)
11036 {
11037#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
11038 /*
11039 * INS/OUTS - I/O String instruction.
11040 *
11041 * Use instruction-information if available, otherwise fall back on
11042 * interpreting the instruction.
11043 */
11044 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
11045 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
11046 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
11047 {
11048 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11049 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
11050 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11051 AssertRCReturn(rc2, rc2);
11052 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
11053 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
11054 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
11055 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
11056 if (fIOWrite)
11057 {
11058 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
11059 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
11060 }
11061 else
11062 {
11063 /*
11064 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
11065 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
11066 * See Intel Instruction spec. for "INS".
11067 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
11068 */
11069 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
11070 }
11071 }
11072 else
11073 {
11074 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
11075 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11076 AssertRCReturn(rc2, rc2);
11077 rcStrict = IEMExecOne(pVCpu);
11078 }
11079 /** @todo IEM needs to be setting these flags somehow. */
11080 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11081 fUpdateRipAlready = true;
11082#else
11083 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11084 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
11085 if (RT_SUCCESS(rcStrict))
11086 {
11087 if (fIOWrite)
11088 {
11089 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
11090 (DISCPUMODE)pDis->uAddrMode, cbValue);
11091 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
11092 }
11093 else
11094 {
11095 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
11096 (DISCPUMODE)pDis->uAddrMode, cbValue);
11097 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
11098 }
11099 }
11100 else
11101 {
11102 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
11103 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
11104 }
11105#endif
11106 }
11107 else
11108 {
11109 /*
11110 * IN/OUT - I/O instruction.
11111 */
11112 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
11113 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
11114 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
11115 if (fIOWrite)
11116 {
11117 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
11118 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11119 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11120 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
11121 }
11122 else
11123 {
11124 uint32_t u32Result = 0;
11125 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
11126 if (IOM_SUCCESS(rcStrict))
11127 {
11128 /* Save result of I/O IN instr. in AL/AX/EAX. */
11129 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
11130 }
11131 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11132 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11133 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
11134 }
11135 }
11136
11137 if (IOM_SUCCESS(rcStrict))
11138 {
11139 if (!fUpdateRipAlready)
11140 {
11141 pMixedCtx->rip += cbInstr;
11142 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11143 }
11144
11145 /*
11146 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
11147 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
11148 */
11149 if (fIOString)
11150 {
11151 /** @todo Single-step for INS/OUTS with REP prefix? */
11152 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
11153 }
11154 else if (fStepping)
11155 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11156
11157 /*
11158 * If any I/O breakpoints are armed, we need to check if one triggered
11159 * and take appropriate action.
11160 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
11161 */
11162 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11163 AssertRCReturn(rc2, rc2);
11164
11165 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
11166 * execution engines about whether hyper BPs and such are pending. */
11167 uint32_t const uDr7 = pMixedCtx->dr[7];
11168 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
11169 && X86_DR7_ANY_RW_IO(uDr7)
11170 && (pMixedCtx->cr4 & X86_CR4_DE))
11171 || DBGFBpIsHwIoArmed(pVM)))
11172 {
11173 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
11174
11175 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
11176 VMMRZCallRing3Disable(pVCpu);
11177 HM_DISABLE_PREEMPT_IF_NEEDED();
11178
11179 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
11180
11181 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
11182 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
11183 {
11184 /* Raise #DB. */
11185 if (fIsGuestDbgActive)
11186 ASMSetDR6(pMixedCtx->dr[6]);
11187 if (pMixedCtx->dr[7] != uDr7)
11188 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11189
11190 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
11191 }
11192 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
11193 else if ( rcStrict2 != VINF_SUCCESS
11194 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
11195 rcStrict = rcStrict2;
11196
11197 HM_RESTORE_PREEMPT_IF_NEEDED();
11198 VMMRZCallRing3Enable(pVCpu);
11199 }
11200 }
11201
11202#ifdef DEBUG
11203 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11204 Assert(!fIOWrite);
11205 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11206 Assert(fIOWrite);
11207 else
11208 {
11209 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
11210 * statuses, that the VMM device and some others may return. See
11211 * IOM_SUCCESS() for guidance. */
11212 AssertMsg( RT_FAILURE(rcStrict)
11213 || rcStrict == VINF_SUCCESS
11214 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
11215 || rcStrict == VINF_EM_DBG_BREAKPOINT
11216 || rcStrict == VINF_EM_RAW_GUEST_TRAP
11217 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11218 }
11219#endif
11220
11221 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
11222 return VBOXSTRICTRC_TODO(rcStrict);
11223}
11224
11225
11226/**
11227 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
11228 * VM-exit.
11229 */
11230HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11231{
11232 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11233
11234 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
11235 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11236 AssertRCReturn(rc, rc);
11237 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
11238 {
11239 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
11240 AssertRCReturn(rc, rc);
11241 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
11242 {
11243 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
11244
11245 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
11246 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
11247
11248 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
11249 Assert(!pVCpu->hm.s.Event.fPending);
11250 pVCpu->hm.s.Event.fPending = true;
11251 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
11252 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
11253 AssertRCReturn(rc, rc);
11254 if (fErrorCodeValid)
11255 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
11256 else
11257 pVCpu->hm.s.Event.u32ErrCode = 0;
11258 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
11259 && uVector == X86_XCPT_PF)
11260 {
11261 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
11262 }
11263
11264 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
11265 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11266 return VINF_EM_RAW_INJECT_TRPM_EVENT;
11267 }
11268 }
11269
11270 /** @todo Emulate task switch someday, currently just going back to ring-3 for
11271 * emulation. */
11272 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11273 return VERR_EM_INTERPRETER;
11274}
11275
11276
11277/**
11278 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
11279 */
11280HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11281{
11282 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11283 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
11284 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
11285 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11286 AssertRCReturn(rc, rc);
11287 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
11288 return VINF_EM_DBG_STEPPED;
11289}
11290
11291
11292/**
11293 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
11294 */
11295HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11296{
11297 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11298
11299 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11300 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11301 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11302 {
11303 if (rc == VINF_HM_DOUBLE_FAULT)
11304 rc = VINF_SUCCESS;
11305 return rc;
11306 }
11307
11308#if 0
11309 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
11310 * just sync the whole thing. */
11311 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11312#else
11313 /* Aggressive state sync. for now. */
11314 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11315 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11316 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11317#endif
11318 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11319 AssertRCReturn(rc, rc);
11320
11321 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
11322 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
11323 switch (uAccessType)
11324 {
11325 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
11326 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
11327 {
11328 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
11329 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != 0x80,
11330 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
11331
11332 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
11333 GCPhys &= PAGE_BASE_GC_MASK;
11334 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
11335 PVM pVM = pVCpu->CTX_SUFF(pVM);
11336 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
11337 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
11338
11339 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
11340 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
11341 CPUMCTX2CORE(pMixedCtx), GCPhys);
11342 rc = VBOXSTRICTRC_VAL(rc2);
11343 Log4(("ApicAccess rc=%d\n", rc));
11344 if ( rc == VINF_SUCCESS
11345 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11346 || rc == VERR_PAGE_NOT_PRESENT)
11347 {
11348 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11349 | HM_CHANGED_GUEST_RSP
11350 | HM_CHANGED_GUEST_RFLAGS
11351 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11352 rc = VINF_SUCCESS;
11353 }
11354 break;
11355 }
11356
11357 default:
11358 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
11359 rc = VINF_EM_RAW_EMULATE_INSTR;
11360 break;
11361 }
11362
11363 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
11364 if (rc != VINF_SUCCESS)
11365 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
11366 return rc;
11367}
11368
11369
11370/**
11371 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
11372 * VM-exit.
11373 */
11374HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11375{
11376 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11377
11378 /* We should -not- get this VM-exit if the guest's debug registers were active. */
11379 if (pVmxTransient->fWasGuestDebugStateActive)
11380 {
11381 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11382 HMVMX_RETURN_UNEXPECTED_EXIT();
11383 }
11384
11385 int rc = VERR_INTERNAL_ERROR_5;
11386 if ( !DBGFIsStepping(pVCpu)
11387 && !pVCpu->hm.s.fSingleInstruction
11388 && !pVmxTransient->fWasHyperDebugStateActive)
11389 {
11390 /* Don't intercept MOV DRx and #DB any more. */
11391 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
11392 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11393 AssertRCReturn(rc, rc);
11394
11395 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11396 {
11397#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11398 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
11399 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
11400 AssertRCReturn(rc, rc);
11401#endif
11402 }
11403
11404 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
11405 VMMRZCallRing3Disable(pVCpu);
11406 HM_DISABLE_PREEMPT_IF_NEEDED();
11407
11408 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
11409 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
11410 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
11411
11412 HM_RESTORE_PREEMPT_IF_NEEDED();
11413 VMMRZCallRing3Enable(pVCpu);
11414
11415#ifdef VBOX_WITH_STATISTICS
11416 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11417 AssertRCReturn(rc, rc);
11418 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11419 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11420 else
11421 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11422#endif
11423 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
11424 return VINF_SUCCESS;
11425 }
11426
11427 /*
11428 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
11429 * Update the segment registers and DR7 from the CPU.
11430 */
11431 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11432 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11433 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11434 AssertRCReturn(rc, rc);
11435 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11436
11437 PVM pVM = pVCpu->CTX_SUFF(pVM);
11438 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11439 {
11440 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11441 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
11442 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
11443 if (RT_SUCCESS(rc))
11444 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11445 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11446 }
11447 else
11448 {
11449 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11450 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
11451 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
11452 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11453 }
11454
11455 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
11456 if (RT_SUCCESS(rc))
11457 {
11458 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11459 AssertRCReturn(rc2, rc2);
11460 }
11461 return rc;
11462}
11463
11464
11465/**
11466 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
11467 * Conditional VM-exit.
11468 */
11469HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11470{
11471 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11472 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11473
11474 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11475 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11476 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11477 {
11478 if (rc == VINF_HM_DOUBLE_FAULT)
11479 rc = VINF_SUCCESS;
11480 return rc;
11481 }
11482
11483 RTGCPHYS GCPhys = 0;
11484 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11485
11486#if 0
11487 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11488#else
11489 /* Aggressive state sync. for now. */
11490 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11491 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11492 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11493#endif
11494 AssertRCReturn(rc, rc);
11495
11496 /*
11497 * If we succeed, resume guest execution.
11498 * If we fail in interpreting the instruction because we couldn't get the guest physical address
11499 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
11500 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
11501 * weird case. See @bugref{6043}.
11502 */
11503 PVM pVM = pVCpu->CTX_SUFF(pVM);
11504 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
11505 rc = VBOXSTRICTRC_VAL(rc2);
11506 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
11507 if ( rc == VINF_SUCCESS
11508 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11509 || rc == VERR_PAGE_NOT_PRESENT)
11510 {
11511 /* Successfully handled MMIO operation. */
11512 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11513 | HM_CHANGED_GUEST_RSP
11514 | HM_CHANGED_GUEST_RFLAGS
11515 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11516 rc = VINF_SUCCESS;
11517 }
11518 return rc;
11519}
11520
11521
11522/**
11523 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
11524 * VM-exit.
11525 */
11526HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11527{
11528 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11529 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11530
11531 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11532 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11533 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11534 {
11535 if (rc == VINF_HM_DOUBLE_FAULT)
11536 rc = VINF_SUCCESS;
11537 return rc;
11538 }
11539
11540 RTGCPHYS GCPhys = 0;
11541 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11542 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11543#if 0
11544 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11545#else
11546 /* Aggressive state sync. for now. */
11547 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11548 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11549 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11550#endif
11551 AssertRCReturn(rc, rc);
11552
11553 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
11554 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
11555
11556 RTGCUINT uErrorCode = 0;
11557 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
11558 uErrorCode |= X86_TRAP_PF_ID;
11559 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
11560 uErrorCode |= X86_TRAP_PF_RW;
11561 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
11562 uErrorCode |= X86_TRAP_PF_P;
11563
11564 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
11565
11566 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
11567 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11568
11569 /* Handle the pagefault trap for the nested shadow table. */
11570 PVM pVM = pVCpu->CTX_SUFF(pVM);
11571 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
11572 TRPMResetTrap(pVCpu);
11573
11574 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
11575 if ( rc == VINF_SUCCESS
11576 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11577 || rc == VERR_PAGE_NOT_PRESENT)
11578 {
11579 /* Successfully synced our nested page tables. */
11580 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
11581 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11582 | HM_CHANGED_GUEST_RSP
11583 | HM_CHANGED_GUEST_RFLAGS);
11584 return VINF_SUCCESS;
11585 }
11586
11587 Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
11588 return rc;
11589}
11590
11591/** @} */
11592
11593/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11594/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
11595/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11596
11597/** @name VM-exit exception handlers.
11598 * @{
11599 */
11600
11601/**
11602 * VM-exit exception handler for #MF (Math Fault: floating point exception).
11603 */
11604static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11605{
11606 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11607 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
11608
11609 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11610 AssertRCReturn(rc, rc);
11611
11612 if (!(pMixedCtx->cr0 & X86_CR0_NE))
11613 {
11614 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
11615 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
11616
11617 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
11618 * provides VM-exit instruction length. If this causes problem later,
11619 * disassemble the instruction like it's done on AMD-V. */
11620 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11621 AssertRCReturn(rc2, rc2);
11622 return rc;
11623 }
11624
11625 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11626 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11627 return rc;
11628}
11629
11630
11631/**
11632 * VM-exit exception handler for #BP (Breakpoint exception).
11633 */
11634static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11635{
11636 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11637 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
11638
11639 /** @todo Try optimize this by not saving the entire guest state unless
11640 * really needed. */
11641 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11642 AssertRCReturn(rc, rc);
11643
11644 PVM pVM = pVCpu->CTX_SUFF(pVM);
11645 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11646 if (rc == VINF_EM_RAW_GUEST_TRAP)
11647 {
11648 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11649 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11650 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11651 AssertRCReturn(rc, rc);
11652
11653 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11654 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11655 }
11656
11657 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
11658 return rc;
11659}
11660
11661
11662/**
11663 * VM-exit exception handler for #DB (Debug exception).
11664 */
11665static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11666{
11667 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11668 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
11669 Log6(("XcptDB\n"));
11670
11671 /*
11672 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
11673 * for processing.
11674 */
11675 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11676 AssertRCReturn(rc, rc);
11677
11678 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
11679 uint64_t uDR6 = X86_DR6_INIT_VAL;
11680 uDR6 |= ( pVmxTransient->uExitQualification
11681 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
11682
11683 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
11684 if (rc == VINF_EM_RAW_GUEST_TRAP)
11685 {
11686 /*
11687 * The exception was for the guest. Update DR6, DR7.GD and
11688 * IA32_DEBUGCTL.LBR before forwarding it.
11689 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
11690 */
11691 VMMRZCallRing3Disable(pVCpu);
11692 HM_DISABLE_PREEMPT_IF_NEEDED();
11693
11694 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
11695 pMixedCtx->dr[6] |= uDR6;
11696 if (CPUMIsGuestDebugStateActive(pVCpu))
11697 ASMSetDR6(pMixedCtx->dr[6]);
11698
11699 HM_RESTORE_PREEMPT_IF_NEEDED();
11700 VMMRZCallRing3Enable(pVCpu);
11701
11702 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11703 AssertRCReturn(rc, rc);
11704
11705 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
11706 pMixedCtx->dr[7] &= ~X86_DR7_GD;
11707
11708 /* Paranoia. */
11709 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
11710 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
11711
11712 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
11713 AssertRCReturn(rc, rc);
11714
11715 /*
11716 * Raise #DB in the guest.
11717 *
11718 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
11719 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
11720 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
11721 *
11722 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
11723 */
11724 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11725 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11726 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11727 AssertRCReturn(rc, rc);
11728 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11729 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11730 return VINF_SUCCESS;
11731 }
11732
11733 /*
11734 * Not a guest trap, must be a hypervisor related debug event then.
11735 * Update DR6 in case someone is interested in it.
11736 */
11737 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
11738 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
11739 CPUMSetHyperDR6(pVCpu, uDR6);
11740
11741 return rc;
11742}
11743
11744
11745/**
11746 * VM-exit exception handler for #NM (Device-not-available exception: floating
11747 * point exception).
11748 */
11749static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11750{
11751 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11752
11753 /* We require CR0 and EFER. EFER is always up-to-date. */
11754 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11755 AssertRCReturn(rc, rc);
11756
11757 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11758 VMMRZCallRing3Disable(pVCpu);
11759 HM_DISABLE_PREEMPT_IF_NEEDED();
11760
11761 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11762 if (pVmxTransient->fWasGuestFPUStateActive)
11763 {
11764 rc = VINF_EM_RAW_GUEST_TRAP;
11765 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11766 }
11767 else
11768 {
11769#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11770 Assert(!pVmxTransient->fWasGuestFPUStateActive);
11771#endif
11772 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11773 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11774 }
11775
11776 HM_RESTORE_PREEMPT_IF_NEEDED();
11777 VMMRZCallRing3Enable(pVCpu);
11778
11779 if (rc == VINF_SUCCESS)
11780 {
11781 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11782 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11783 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11784 pVCpu->hm.s.fUseGuestFpu = true;
11785 }
11786 else
11787 {
11788 /* Forward #NM to the guest. */
11789 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11790 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11791 AssertRCReturn(rc, rc);
11792 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11793 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
11794 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11795 }
11796
11797 return VINF_SUCCESS;
11798}
11799
11800
11801/**
11802 * VM-exit exception handler for #GP (General-protection exception).
11803 *
11804 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11805 */
11806static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11807{
11808 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11809 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11810
11811 int rc = VERR_INTERNAL_ERROR_5;
11812 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11813 {
11814#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11815 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11816 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11817 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11818 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11819 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11820 AssertRCReturn(rc, rc);
11821 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
11822 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
11823 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11824 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11825 return rc;
11826#else
11827 /* We don't intercept #GP. */
11828 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11829 NOREF(pVmxTransient);
11830 return VERR_VMX_UNEXPECTED_EXCEPTION;
11831#endif
11832 }
11833
11834 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11835 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11836
11837 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11838 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11839 AssertRCReturn(rc, rc);
11840
11841 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11842 uint32_t cbOp = 0;
11843 PVM pVM = pVCpu->CTX_SUFF(pVM);
11844 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11845 if (RT_SUCCESS(rc))
11846 {
11847 rc = VINF_SUCCESS;
11848 Assert(cbOp == pDis->cbInstr);
11849 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11850 switch (pDis->pCurInstr->uOpcode)
11851 {
11852 case OP_CLI:
11853 {
11854 pMixedCtx->eflags.Bits.u1IF = 0;
11855 pMixedCtx->eflags.Bits.u1RF = 0;
11856 pMixedCtx->rip += pDis->cbInstr;
11857 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11858 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11859 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11860 break;
11861 }
11862
11863 case OP_STI:
11864 {
11865 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
11866 pMixedCtx->eflags.Bits.u1IF = 1;
11867 pMixedCtx->eflags.Bits.u1RF = 0;
11868 pMixedCtx->rip += pDis->cbInstr;
11869 if (!fOldIF)
11870 {
11871 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11872 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11873 }
11874 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11875 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11876 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11877 break;
11878 }
11879
11880 case OP_HLT:
11881 {
11882 rc = VINF_EM_HALT;
11883 pMixedCtx->rip += pDis->cbInstr;
11884 pMixedCtx->eflags.Bits.u1RF = 0;
11885 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11886 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11887 break;
11888 }
11889
11890 case OP_POPF:
11891 {
11892 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11893 uint32_t cbParm;
11894 uint32_t uMask;
11895 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11896 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11897 {
11898 cbParm = 4;
11899 uMask = 0xffffffff;
11900 }
11901 else
11902 {
11903 cbParm = 2;
11904 uMask = 0xffff;
11905 }
11906
11907 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
11908 RTGCPTR GCPtrStack = 0;
11909 X86EFLAGS Eflags;
11910 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11911 &GCPtrStack);
11912 if (RT_SUCCESS(rc))
11913 {
11914 Assert(sizeof(Eflags.u32) >= cbParm);
11915 Eflags.u32 = 0;
11916 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11917 }
11918 if (RT_FAILURE(rc))
11919 {
11920 rc = VERR_EM_INTERPRETER;
11921 break;
11922 }
11923 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11924 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
11925 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11926 pMixedCtx->esp += cbParm;
11927 pMixedCtx->esp &= uMask;
11928 pMixedCtx->rip += pDis->cbInstr;
11929 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11930 | HM_CHANGED_GUEST_RSP
11931 | HM_CHANGED_GUEST_RFLAGS);
11932 /* Generate a pending-debug exception when stepping over POPF regardless of how POPF modifies EFLAGS.TF. */
11933 if (fStepping)
11934 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11935
11936 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11937 break;
11938 }
11939
11940 case OP_PUSHF:
11941 {
11942 uint32_t cbParm;
11943 uint32_t uMask;
11944 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11945 {
11946 cbParm = 4;
11947 uMask = 0xffffffff;
11948 }
11949 else
11950 {
11951 cbParm = 2;
11952 uMask = 0xffff;
11953 }
11954
11955 /* Get the stack pointer & push the contents of eflags onto the stack. */
11956 RTGCPTR GCPtrStack = 0;
11957 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11958 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11959 if (RT_FAILURE(rc))
11960 {
11961 rc = VERR_EM_INTERPRETER;
11962 break;
11963 }
11964 X86EFLAGS Eflags = pMixedCtx->eflags;
11965 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11966 Eflags.Bits.u1RF = 0;
11967 Eflags.Bits.u1VM = 0;
11968
11969 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11970 if (RT_FAILURE(rc))
11971 {
11972 rc = VERR_EM_INTERPRETER;
11973 break;
11974 }
11975 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11976 pMixedCtx->esp -= cbParm;
11977 pMixedCtx->esp &= uMask;
11978 pMixedCtx->rip += pDis->cbInstr;
11979 pMixedCtx->eflags.Bits.u1RF = 0;
11980 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11981 | HM_CHANGED_GUEST_RSP
11982 | HM_CHANGED_GUEST_RFLAGS);
11983 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11984 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11985 break;
11986 }
11987
11988 case OP_IRET:
11989 {
11990 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11991 * instruction reference. */
11992 RTGCPTR GCPtrStack = 0;
11993 uint32_t uMask = 0xffff;
11994 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11995 uint16_t aIretFrame[3];
11996 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
11997 {
11998 rc = VERR_EM_INTERPRETER;
11999 break;
12000 }
12001 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
12002 &GCPtrStack);
12003 if (RT_SUCCESS(rc))
12004 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
12005 if (RT_FAILURE(rc))
12006 {
12007 rc = VERR_EM_INTERPRETER;
12008 break;
12009 }
12010 pMixedCtx->eip = 0;
12011 pMixedCtx->ip = aIretFrame[0];
12012 pMixedCtx->cs.Sel = aIretFrame[1];
12013 pMixedCtx->cs.ValidSel = aIretFrame[1];
12014 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
12015 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
12016 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
12017 pMixedCtx->sp += sizeof(aIretFrame);
12018 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12019 | HM_CHANGED_GUEST_SEGMENT_REGS
12020 | HM_CHANGED_GUEST_RSP
12021 | HM_CHANGED_GUEST_RFLAGS);
12022 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
12023 if (fStepping)
12024 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
12025 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
12026 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
12027 break;
12028 }
12029
12030 case OP_INT:
12031 {
12032 uint16_t uVector = pDis->Param1.uValue & 0xff;
12033 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
12034 /* INT clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
12035 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
12036 break;
12037 }
12038
12039 case OP_INTO:
12040 {
12041 if (pMixedCtx->eflags.Bits.u1OF)
12042 {
12043 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
12044 /* INTO clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
12045 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
12046 }
12047 else
12048 {
12049 pMixedCtx->eflags.Bits.u1RF = 0;
12050 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12051 }
12052 break;
12053 }
12054
12055 default:
12056 {
12057 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
12058 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
12059 EMCODETYPE_SUPERVISOR);
12060 rc = VBOXSTRICTRC_VAL(rc2);
12061 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
12062 /** @todo We have to set pending-debug exceptions here when the guest is
12063 * single-stepping depending on the instruction that was interpreted. */
12064 Log4(("#GP rc=%Rrc\n", rc));
12065 break;
12066 }
12067 }
12068 }
12069 else
12070 rc = VERR_EM_INTERPRETER;
12071
12072 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
12073 ("#GP Unexpected rc=%Rrc\n", rc));
12074 return rc;
12075}
12076
12077
12078#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12079/**
12080 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
12081 * the exception reported in the VMX transient structure back into the VM.
12082 *
12083 * @remarks Requires uExitIntInfo in the VMX transient structure to be
12084 * up-to-date.
12085 */
12086static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12087{
12088 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12089
12090 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
12091 hmR0VmxCheckExitDueToEventDelivery(). */
12092 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12093 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12094 AssertRCReturn(rc, rc);
12095 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
12096
12097#ifdef DEBUG_ramshankar
12098 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12099 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12100 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
12101#endif
12102
12103 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12104 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12105 return VINF_SUCCESS;
12106}
12107#endif
12108
12109
12110/**
12111 * VM-exit exception handler for #PF (Page-fault exception).
12112 */
12113static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12114{
12115 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12116 PVM pVM = pVCpu->CTX_SUFF(pVM);
12117 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12118 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12119 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12120 AssertRCReturn(rc, rc);
12121
12122#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
12123 if (pVM->hm.s.fNestedPaging)
12124 {
12125 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
12126 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
12127 {
12128 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12129 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12130 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
12131 }
12132 else
12133 {
12134 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12135 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12136 Log4(("Pending #DF due to vectoring #PF. NP\n"));
12137 }
12138 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12139 return rc;
12140 }
12141#else
12142 Assert(!pVM->hm.s.fNestedPaging);
12143 NOREF(pVM);
12144#endif
12145
12146 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
12147 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
12148 if (pVmxTransient->fVectoringPF)
12149 {
12150 Assert(pVCpu->hm.s.Event.fPending);
12151 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12152 }
12153
12154 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12155 AssertRCReturn(rc, rc);
12156
12157 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
12158 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
12159
12160 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
12161 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
12162 (RTGCPTR)pVmxTransient->uExitQualification);
12163
12164 Log4(("#PF: rc=%Rrc\n", rc));
12165 if (rc == VINF_SUCCESS)
12166 {
12167 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
12168 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
12169 * memory? We don't update the whole state here... */
12170 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12171 | HM_CHANGED_GUEST_RSP
12172 | HM_CHANGED_GUEST_RFLAGS
12173 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12174 TRPMResetTrap(pVCpu);
12175 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
12176 return rc;
12177 }
12178
12179 if (rc == VINF_EM_RAW_GUEST_TRAP)
12180 {
12181 if (!pVmxTransient->fVectoringDoublePF)
12182 {
12183 /* It's a guest page fault and needs to be reflected to the guest. */
12184 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
12185 TRPMResetTrap(pVCpu);
12186 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
12187 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12188 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12189 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
12190 }
12191 else
12192 {
12193 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12194 TRPMResetTrap(pVCpu);
12195 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
12196 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12197 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
12198 }
12199
12200 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12201 return VINF_SUCCESS;
12202 }
12203
12204 TRPMResetTrap(pVCpu);
12205 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
12206 return rc;
12207}
12208
12209/** @} */
12210
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette